diff options
author | Jon Marius Venstad <venstad@gmail.com> | 2020-11-03 17:14:28 +0100 |
---|---|---|
committer | Jon Marius Venstad <venstad@gmail.com> | 2020-11-03 17:14:28 +0100 |
commit | 7a17516b491c1bb4a248b4cf7d3488fe93eabe34 (patch) | |
tree | fc82a314db40d93aa63ef5c46a5443193a43aaef /vespaclient-container-plugin | |
parent | c3c0c67621ec2ad422e69159fe46698e45e2cac1 (diff) |
Remove old /document/v1 handler
Diffstat (limited to 'vespaclient-container-plugin')
17 files changed, 10 insertions, 2640 deletions
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java deleted file mode 100644 index 325c5492776..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/LocalDataVisitorHandler.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.restapi; - -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.document.json.JsonWriter; -import com.yahoo.documentapi.DumpVisitorDataHandler; -import com.yahoo.exception.ExceptionUtils; - -import java.nio.charset.StandardCharsets; - -/** - * Handling data from visit. - * - * @author dybis - */ -class LocalDataVisitorHandler extends DumpVisitorDataHandler { - - StringBuilder commaSeparatedJsonDocuments = new StringBuilder(); - final StringBuilder errors = new StringBuilder(); - - private boolean isFirst = true; - private final Object monitor = new Object(); - - String getErrors() { - return errors.toString(); - } - - String getCommaSeparatedJsonDocuments() { - return commaSeparatedJsonDocuments.toString(); - } - - @Override - public void onDocument(Document document, long l) { - try { - final String docJson = new String(JsonWriter.toByteArray(document), StandardCharsets.UTF_8.name()); - synchronized (monitor) { - if (!isFirst) { - commaSeparatedJsonDocuments.append(","); - } - isFirst = false; - commaSeparatedJsonDocuments.append(docJson); - } - } catch (Exception e) { - synchronized (monitor) { - errors.append(ExceptionUtils.getStackTraceAsString(e)).append("\n"); - } - } - } - - // TODO: Not sure if we should support removal or not. Do nothing here maybe? - @Override - public void onRemove(DocumentId documentId) { - try { - final String removeJson = new String(JsonWriter.documentRemove(documentId), StandardCharsets.UTF_8.name()); - synchronized (monitor) { - if (!isFirst) { - commaSeparatedJsonDocuments.append(","); - } - isFirst = false; - commaSeparatedJsonDocuments.append(removeJson); - } - } catch (Exception e) { - synchronized (monitor) { - errors.append(ExceptionUtils.getStackTraceAsString(e)).append("\n"); - } - } - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java deleted file mode 100644 index 848fe4b5726..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.restapi; - -import com.yahoo.vespaxmlparser.FeedOperation; - -import java.util.Optional; - -/** - * Abstract the backend stuff for the REST API, such as retrieving or updating documents. - * - * @author Haakon Dybdahl - */ -public interface OperationHandler { - - class VisitResult { - - public final Optional<String> token; - public final String documentsAsJsonList; - - public VisitResult(Optional<String> token, String documentsAsJsonList) { - this.token = token; - this.documentsAsJsonList = documentsAsJsonList; - } - } - - class VisitOptions { - public final Optional<String> cluster; - public final Optional<String> continuation; - public final Optional<Integer> wantedDocumentCount; - public final Optional<String> fieldSet; - public final Optional<Integer> concurrency; - public final Optional<String> bucketSpace; - - private VisitOptions(Builder builder) { - this.cluster = Optional.ofNullable(builder.cluster); - this.continuation = Optional.ofNullable(builder.continuation); - this.wantedDocumentCount = Optional.ofNullable(builder.wantedDocumentCount); - this.fieldSet = Optional.ofNullable(builder.fieldSet); - this.concurrency = Optional.ofNullable(builder.concurrency); - this.bucketSpace = Optional.ofNullable(builder.bucketSpace); - } - - public static class Builder { - String cluster; - String continuation; - Integer wantedDocumentCount; - String fieldSet; - Integer concurrency; - String bucketSpace; - - public Builder cluster(String cluster) { - this.cluster = cluster; - return this; - } - - public Builder continuation(String continuation) { - this.continuation = continuation; - return this; - } - - public Builder wantedDocumentCount(Integer count) { - this.wantedDocumentCount = count; - return this; - } - - public Builder fieldSet(String fieldSet) { - this.fieldSet = fieldSet; - return this; - } - - public Builder concurrency(Integer concurrency) { - this.concurrency = concurrency; - return this; - } - - public Builder bucketSpace(String bucketSpace) { - this.bucketSpace = bucketSpace; - return this; - } - - public VisitOptions build() { - return new VisitOptions(this); - } - } - - public static Builder builder() { - return new Builder(); - } - } - - VisitResult visit(RestUri restUri, String documentSelection, VisitOptions options) throws RestApiException; - - void put(RestUri restUri, FeedOperation data, Optional<String> route) throws RestApiException; - - void update(RestUri restUri, FeedOperation data, Optional<String> route) throws RestApiException; - - void delete(RestUri restUri, String condition, Optional<String> route) throws RestApiException; - - Optional<String> get(RestUri restUri) throws RestApiException; - - default Optional<String> get(RestUri restUri, Optional<String> fieldSet) throws RestApiException { - return get(restUri); - } - - default Optional<String> get(RestUri restUri, Optional<String> fieldSet, Optional<String> cluster) throws RestApiException { - return get(restUri, fieldSet); - } - - /** Called just before this is disposed of */ - default void shutdown() {} - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java deleted file mode 100644 index 3d3a8fc52ad..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ /dev/null @@ -1,460 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.restapi; - -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentRemove; -import com.yahoo.document.FixedBucketSpaces; -import com.yahoo.document.TestAndSetCondition; -import com.yahoo.document.fieldset.AllFields; -import com.yahoo.document.json.JsonWriter; -import com.yahoo.document.DocumentPut; -import com.yahoo.documentapi.DocumentAccess; -import com.yahoo.documentapi.DocumentAccessException; -import com.yahoo.documentapi.ProgressToken; -import com.yahoo.documentapi.SyncParameters; -import com.yahoo.documentapi.SyncSession; -import com.yahoo.documentapi.VisitorControlHandler; -import com.yahoo.documentapi.VisitorParameters; -import com.yahoo.documentapi.VisitorSession; -import com.yahoo.documentapi.messagebus.MessageBusSyncSession; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.metrics.DocumentApiMetrics; -import com.yahoo.documentapi.metrics.DocumentOperationStatus; -import com.yahoo.documentapi.metrics.DocumentOperationType; -import com.yahoo.exception.ExceptionUtils; -import com.yahoo.messagebus.StaticThrottlePolicy; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.vespaclient.ClusterDef; -import com.yahoo.vespaxmlparser.FeedOperation; -import com.yahoo.yolean.concurrent.ConcurrentResourcePool; -import com.yahoo.yolean.concurrent.ResourceFactory; - -import java.io.ByteArrayOutputStream; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Sends operations to messagebus via document api. - * - * @author dybis - */ -public class OperationHandlerImpl implements OperationHandler { - - public interface ClusterEnumerator { - List<ClusterDef> enumerateClusters(); - } - - public interface BucketSpaceResolver { - Optional<String> clusterBucketSpaceFromDocumentType(String clusterId, String docType); - } - - public static class BucketSpaceRoute { - private final String clusterRoute; - private final String bucketSpace; - - public BucketSpaceRoute(String clusterRoute, String bucketSpace) { - this.clusterRoute = clusterRoute; - this.bucketSpace = bucketSpace; - } - - public String getClusterRoute() { - return clusterRoute; - } - - public String getBucketSpace() { - return bucketSpace; - } - } - - public static final int VISIT_TIMEOUT_MS = 120000; - public static final int WANTED_DOCUMENT_COUNT_UPPER_BOUND = 1000; // Approximates the max default size of a bucket - public static final int CONCURRENCY_UPPER_BOUND = 100; - private final DocumentAccess documentAccess; - private final DocumentApiMetrics metricsHelper; - private final ClusterEnumerator clusterEnumerator; - private final BucketSpaceResolver bucketSpaceResolver; - - private static final class SyncSessionFactory extends ResourceFactory<SyncSession> { - private final DocumentAccess documentAccess; - SyncSessionFactory(DocumentAccess documentAccess) { - this.documentAccess = documentAccess; - } - @Override - public SyncSession create() { - return documentAccess.createSyncSession(new SyncParameters.Builder().build()); - } - } - - private final ConcurrentResourcePool<SyncSession> syncSessions; - - public OperationHandlerImpl(DocumentAccess documentAccess, ClusterEnumerator clusterEnumerator, - BucketSpaceResolver bucketSpaceResolver, MetricReceiver metricReceiver) { - this.documentAccess = documentAccess; - this.clusterEnumerator = clusterEnumerator; - this.bucketSpaceResolver = bucketSpaceResolver; - syncSessions = new ConcurrentResourcePool<>(new SyncSessionFactory(documentAccess)); - metricsHelper = new DocumentApiMetrics(metricReceiver, "documentV1"); - } - - @Override - public void shutdown() { - for (SyncSession session : syncSessions) { - session.destroy(); - } - documentAccess.shutdown(); - } - - private static final int HTTP_STATUS_BAD_REQUEST = 400; - private static final int HTTP_STATUS_INSUFFICIENT_STORAGE = 507; - private static final int HTTP_PRECONDITION_FAILED = 412; - - public static int getHTTPStatusCode(Set<Integer> errorCodes) { - if (errorCodes.size() == 1 && errorCodes.contains(DocumentProtocol.ERROR_NO_SPACE)) { - return HTTP_STATUS_INSUFFICIENT_STORAGE; - } - if (errorCodes.contains(DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED)) { - return HTTP_PRECONDITION_FAILED; - } - return HTTP_STATUS_BAD_REQUEST; - } - - private static Response createErrorResponse(DocumentAccessException documentException, RestUri restUri) { - if (documentException.hasConditionNotMetError()) { - return Response.createErrorResponse(getHTTPStatusCode(documentException.getErrorCodes()), "Condition did not match document.", - restUri, RestUri.apiErrorCodes.DOCUMENT_CONDITION_NOT_MET); - } - return Response.createErrorResponse(getHTTPStatusCode(documentException.getErrorCodes()), documentException.getMessage(), restUri, - RestUri.apiErrorCodes.DOCUMENT_EXCEPTION); - } - - @Override - public VisitResult visit(RestUri restUri, String documentSelection, VisitOptions options) throws RestApiException { - VisitorParameters visitorParameters = createVisitorParameters(restUri, documentSelection, options); - - VisitorControlHandler visitorControlHandler = new VisitorControlHandler(); - visitorParameters.setControlHandler(visitorControlHandler); - LocalDataVisitorHandler localDataVisitorHandler = new LocalDataVisitorHandler(); - visitorParameters.setLocalDataHandler(localDataVisitorHandler); - - final VisitorSession visitorSession; - try { - visitorSession = documentAccess.createVisitorSession(visitorParameters); - // Not sure if this line is required - visitorControlHandler.setSession(visitorSession); - } catch (Exception e) { - throw new RestApiException(Response.createErrorResponse( - 500, - "Failed during parsing of arguments for visiting: " + ExceptionUtils.getStackTraceAsString(e), - restUri, - RestUri.apiErrorCodes.VISITOR_ERROR)); - } - try { - return doVisit(visitorControlHandler, localDataVisitorHandler, restUri); - } finally { - visitorSession.destroy(); - } - } - - private static void throwIfFatalVisitingError(VisitorControlHandler handler, RestUri restUri) throws RestApiException { - final VisitorControlHandler.Result result = handler.getResult(); - if (result.getCode() == VisitorControlHandler.CompletionCode.TIMEOUT) { - if (! handler.hasVisitedAnyBuckets()) { - throw new RestApiException(Response.createErrorResponse(500, "Timed out", restUri, RestUri.apiErrorCodes.TIME_OUT)); - } // else: some progress has been made, let client continue with new token. - } else if (result.getCode() != VisitorControlHandler.CompletionCode.SUCCESS) { - throw new RestApiException(Response.createErrorResponse(400, result.toString(), RestUri.apiErrorCodes.VISITOR_ERROR)); - } - } - - private VisitResult doVisit(VisitorControlHandler visitorControlHandler, - LocalDataVisitorHandler localDataVisitorHandler, - RestUri restUri) throws RestApiException { - try { - visitorControlHandler.waitUntilDone(); // VisitorParameters' session timeout implicitly triggers timeout failures. - throwIfFatalVisitingError(visitorControlHandler, restUri); - } catch (InterruptedException e) { - throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.INTERRUPTED)); - } - if (localDataVisitorHandler.getErrors().isEmpty()) { - Optional<String> continuationToken; - if (! visitorControlHandler.getProgress().isFinished()) { - continuationToken = Optional.of(visitorControlHandler.getProgress().serializeToString()); - } else { - continuationToken = Optional.empty(); - } - return new VisitResult(continuationToken, localDataVisitorHandler.getCommaSeparatedJsonDocuments()); - } - throw new RestApiException(Response.createErrorResponse(500, localDataVisitorHandler.getErrors(), restUri, RestUri.apiErrorCodes.UNSPECIFIED)); - } - - private void setRoute(SyncSession session, Optional<String> route) throws RestApiException { - if (! (session instanceof MessageBusSyncSession)) { - // Not sure if this ever could happen but better be safe. - throw new RestApiException(Response.createErrorResponse( - 400, "Can not set route since the API is not using message bus.", - RestUri.apiErrorCodes.NO_ROUTE_WHEN_NOT_PART_OF_MESSAGEBUS)); - } - ((MessageBusSyncSession) session).setRoute(route.orElse("default")); - } - - @Override - public void put(RestUri restUri, FeedOperation data, Optional<String> route) throws RestApiException { - SyncSession syncSession = syncSessions.alloc(); - Response response; - try { - Instant startTime = Instant.now(); - DocumentPut put = new DocumentPut(data.getDocument()); - put.setCondition(data.getCondition()); - setRoute(syncSession, route); - syncSession.put(put); - metricsHelper.reportSuccessful(DocumentOperationType.PUT, startTime); - return; - } catch (DocumentAccessException documentException) { - response = createErrorResponse(documentException, restUri); - } catch (Exception e) { - response = Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.INTERNAL_EXCEPTION); - } finally { - syncSessions.free(syncSession); - } - - metricsHelper.reportFailure(DocumentOperationType.PUT, DocumentOperationStatus.fromHttpStatusCode(response.getStatus())); - throw new RestApiException(response); - } - - @Override - public void update(RestUri restUri, FeedOperation data, Optional<String> route) throws RestApiException { - SyncSession syncSession = syncSessions.alloc(); - Response response; - try { - Instant startTime = Instant.now(); - setRoute(syncSession, route); - syncSession.update(data.getDocumentUpdate()); - metricsHelper.reportSuccessful(DocumentOperationType.UPDATE, startTime); - return; - } catch (DocumentAccessException documentException) { - response = createErrorResponse(documentException, restUri); - } catch (Exception e) { - response = Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.INTERNAL_EXCEPTION); - } finally { - syncSessions.free(syncSession); - } - - metricsHelper.reportFailure(DocumentOperationType.UPDATE, DocumentOperationStatus.fromHttpStatusCode(response.getStatus())); - throw new RestApiException(response); - } - - @Override - public void delete(RestUri restUri, String condition, Optional<String> route) throws RestApiException { - SyncSession syncSession = syncSessions.alloc(); - Response response; - try { - Instant startTime = Instant.now(); - DocumentId id = new DocumentId(restUri.generateFullId()); - DocumentRemove documentRemove = new DocumentRemove(id); - setRoute(syncSession, route); - if (condition != null && ! condition.isEmpty()) { - documentRemove.setCondition(new TestAndSetCondition(condition)); - } - syncSession.remove(documentRemove); - metricsHelper.reportSuccessful(DocumentOperationType.REMOVE, startTime); - return; - } catch (DocumentAccessException documentException) { - if (documentException.hasConditionNotMetError()) { - response = Response.createErrorResponse(412, "Condition not met: " + documentException.getMessage(), - restUri, RestUri.apiErrorCodes.DOCUMENT_CONDITION_NOT_MET); - } else { - response = Response.createErrorResponse(400, documentException.getMessage(), restUri, RestUri.apiErrorCodes.DOCUMENT_EXCEPTION); - } - } catch (Exception e) { - response = Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED); - } finally { - syncSessions.free(syncSession); - } - - metricsHelper.reportFailure(DocumentOperationType.REMOVE, DocumentOperationStatus.fromHttpStatusCode(response.getStatus())); - throw new RestApiException(response); - } - - @Override - public Optional<String> get(RestUri restUri, Optional<String> fieldSet, Optional<String> cluster) throws RestApiException { - SyncSession syncSession = syncSessions.alloc(); - // Explicit unary used instead of map() due to unhandled exceptions, blargh. - Optional<String> route = cluster.isPresent() - ? Optional.of(clusterDefToRoute(resolveClusterDef(cluster, clusterEnumerator.enumerateClusters()))) - : Optional.empty(); - setRoute(syncSession, route); - try { - DocumentId id = new DocumentId(restUri.generateFullId()); - final Document document = syncSession.get(id, fieldSet.orElse(restUri.getDocumentType() + ":[document]"), DocumentProtocol.Priority.NORMAL_1); - if (document == null) { - return Optional.empty(); - } - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - JsonWriter jsonWriter = new JsonWriter(outputStream); - jsonWriter.write(document); - return Optional.of(outputStream.toString(StandardCharsets.UTF_8.name())); - - } catch (Exception e) { - throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED)); - } finally { - syncSessions.free(syncSession); - } - } - - @Override - public Optional<String> get(RestUri restUri, Optional<String> fieldSet) throws RestApiException { - return get(restUri, fieldSet, Optional.empty()); - } - - @Override - public Optional<String> get(RestUri restUri) throws RestApiException { - return get(restUri, Optional.empty()); - } - - private static boolean isValidBucketSpace(String spaceName) { - // TODO need bucket space repo in Java as well - return (FixedBucketSpaces.defaultSpace().equals(spaceName) - || FixedBucketSpaces.globalSpace().equals(spaceName)); - } - - protected BucketSpaceRoute resolveBucketSpaceRoute(Optional<String> wantedCluster, - Optional<String> wantedBucketSpace, - RestUri restUri) throws RestApiException { - final List<ClusterDef> clusters = clusterEnumerator.enumerateClusters(); - ClusterDef clusterDef = resolveClusterDef(wantedCluster, clusters); - - String targetBucketSpace; - if (!restUri.isRootOnly()) { - String docType = restUri.getDocumentType(); - Optional<String> resolvedSpace = bucketSpaceResolver.clusterBucketSpaceFromDocumentType(clusterDef.getName(), docType); - if (!resolvedSpace.isPresent()) { - throw new RestApiException(Response.createErrorResponse(400, String.format( - "Document type '%s' in cluster '%s' is not mapped to a known bucket space", docType, clusterDef.getName()), - RestUri.apiErrorCodes.UNKNOWN_BUCKET_SPACE)); - } - targetBucketSpace = resolvedSpace.get(); - } else { - if (wantedBucketSpace.isPresent() && !isValidBucketSpace(wantedBucketSpace.get())) { - // TODO enumerate known bucket spaces from a repo instead of having a fixed set - throw new RestApiException(Response.createErrorResponse(400, String.format( - "Bucket space '%s' is not a known bucket space (expected '%s' or '%s')", - wantedBucketSpace.get(), FixedBucketSpaces.defaultSpace(), FixedBucketSpaces.globalSpace()), - RestUri.apiErrorCodes.UNKNOWN_BUCKET_SPACE)); - } - targetBucketSpace = wantedBucketSpace.orElse(FixedBucketSpaces.defaultSpace()); - } - - return new BucketSpaceRoute(clusterDefToRoute(clusterDef), targetBucketSpace); - } - - protected static ClusterDef resolveClusterDef(Optional<String> wantedCluster, List<ClusterDef> clusters) throws RestApiException { - if (clusters.size() == 0) { - throw new IllegalArgumentException("Your Vespa cluster does not have any content clusters " + - "declared. Visiting feature is not available."); - } - if (! wantedCluster.isPresent()) { - if (clusters.size() != 1) { - String message = "Several clusters exist: " + - clusters.stream().map(c -> "'" + c.getName() + "'").collect(Collectors.joining(", ")) + - ". You must specify one."; - throw new RestApiException(Response.createErrorResponse(400, - message, - RestUri.apiErrorCodes.SEVERAL_CLUSTERS)); - } - return clusters.get(0); - } - - for (ClusterDef clusterDef : clusters) { - if (clusterDef.getName().equals(wantedCluster.get())) { - return clusterDef; - } - } - String message = "Your vespa cluster contains the content clusters " + - clusters.stream().map(c -> "'" + c.getName() + "'").collect(Collectors.joining(", ")) + - ", not '" + wantedCluster.get() + "'. Please select a valid vespa cluster."; - throw new RestApiException(Response.createErrorResponse(400, - message, - RestUri.apiErrorCodes.MISSING_CLUSTER)); - } - - protected static String clusterDefToRoute(ClusterDef clusterDef) { - return "[Storage:cluster=" + clusterDef.getName() + ";clusterconfigid=" + clusterDef.getConfigId() + "]"; - } - - private static String buildAugmentedDocumentSelection(RestUri restUri, String documentSelection) { - if (restUri.isRootOnly()) { - return documentSelection; // May be empty, that's fine. - } - StringBuilder selection = new StringBuilder(); - if (! documentSelection.isEmpty()) { - selection.append("((").append(documentSelection).append(") and "); - } - selection.append(restUri.getDocumentType()).append(" and (id.namespace=='").append(restUri.getNamespace()).append("')"); - if (! documentSelection.isEmpty()) { - selection.append(")"); - } - return selection.toString(); - } - - private static int computeEffectiveConcurrency(Optional<Integer> requestConcurrency) { - int wantedConcurrency = requestConcurrency.orElse(1); - return Math.min(Math.max(wantedConcurrency, 1), CONCURRENCY_UPPER_BOUND); - } - - private VisitorParameters createVisitorParameters( - RestUri restUri, - String documentSelection, - VisitOptions options) - throws RestApiException { - - if (restUri.isRootOnly() && !options.cluster.isPresent()) { - throw new RestApiException(Response.createErrorResponse(400, - "Must set 'cluster' parameter to a valid content cluster id when visiting at a root /document/v1/ level", - RestUri.apiErrorCodes.MISSING_CLUSTER)); - } - - String augmentedSelection = buildAugmentedDocumentSelection(restUri, documentSelection); - - VisitorParameters params = new VisitorParameters(augmentedSelection); - // Only return fieldset that is part of the document, unless we're visiting across all - // document types in which case we can't explicitly state a single document type. - // This matches legacy /visit API and vespa-visit tool behavior. - params.fieldSet(options.fieldSet.orElse( - restUri.isRootOnly() ? AllFields.NAME : restUri.getDocumentType() + ":[document]")); - params.setMaxBucketsPerVisitor(1); - params.setMaxPending(32); - params.setMaxFirstPassHits(1); - params.setMaxTotalHits(options.wantedDocumentCount - .map(n -> Math.min(Math.max(n, 1), WANTED_DOCUMENT_COUNT_UPPER_BOUND)) - .orElse(1)); - params.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount(computeEffectiveConcurrency(options.concurrency))); - params.setToTimestamp(0L); - params.setFromTimestamp(0L); - params.setSessionTimeoutMs(VISIT_TIMEOUT_MS); - - params.visitInconsistentBuckets(true); // TODO document this as part of consistency doc - - BucketSpaceRoute bucketSpaceRoute = resolveBucketSpaceRoute(options.cluster, options.bucketSpace, restUri); - params.setRoute(bucketSpaceRoute.getClusterRoute()); - params.setBucketSpace(bucketSpaceRoute.getBucketSpace()); - - params.setTraceLevel(0); - params.setPriority(DocumentProtocol.Priority.NORMAL_4); - params.setVisitRemoves(false); - - if (options.continuation.isPresent()) { - try { - params.setResumeToken(ProgressToken.fromSerializedString(options.continuation.get())); - } catch (Exception e) { - throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTraceAsString(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED)); - } - } - return params; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/Response.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/Response.java deleted file mode 100644 index 663f77e7eea..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/Response.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.restapi; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.yahoo.container.jdisc.HttpResponse; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.Optional; - -public class Response extends HttpResponse { - - private final static ObjectMapper objectMapper = new ObjectMapper(); - private final String jsonMessage; - - public Response(int code, Optional<ObjectNode> element, Optional<RestUri> restPath) { - super(code); - ObjectNode objectNode = element.orElse(objectMapper.createObjectNode()); - if (restPath.isPresent()) { - objectNode.put("id", restPath.get().generateFullId()); - objectNode.put("pathId", restPath.get().getRawPath()); - } - jsonMessage = objectNode.toString(); - } - - public static Response createErrorResponse(int code, String errorMessage, RestUri.apiErrorCodes errorID) { - return createErrorResponse(code, errorMessage, null, errorID); - } - - public static Response createErrorResponse(int code, String errorMessage, RestUri restUri, RestUri.apiErrorCodes errorID) { - ObjectNode errorNode = objectMapper.createObjectNode(); - errorNode.put("description", errorID.name() + " " + errorMessage); - errorNode.put("id", errorID.value); - - ObjectNode objectNode = objectMapper.createObjectNode(); - objectNode.putArray("errors").add(errorNode); - return new Response(code, Optional.of(objectNode), Optional.ofNullable(restUri)); - } - - @Override - public void render(OutputStream stream) throws IOException { - stream.write(jsonMessage.getBytes(StandardCharsets.UTF_8)); - } - - @Override - public String getContentType() { return "application/json"; } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestApiException.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestApiException.java deleted file mode 100644 index 29843801bee..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestApiException.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.restapi; - -/** - * Exceptions for Rest API - * - * @author dybis - */ -public class RestApiException extends Exception { - - private final Response response; - - public RestApiException(Response response) { - this.response = response; - } - - public Response getResponse() { - return response; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java deleted file mode 100644 index 975075fd2fa..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.restapi; - -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; - -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Optional; -import static com.yahoo.jdisc.Response.Status.*; - -/** - * Represents the request URI with its values. - * - * @author dybis - */ -public class RestUri { - - public static final char NUMBER_STREAMING = 'n'; - public static final char GROUP_STREAMING = 'g'; - public static final String DOCUMENT = "document"; - public static final String V_1 = "v1"; - public static final String ID = "id:"; - - public enum apiErrorCodes { - ERROR_ID_BASIC_USAGE(-1), - ERROR_ID_DECODING_PATH(-2), - VISITOR_ERROR(-3), - NO_ROUTE_WHEN_NOT_PART_OF_MESSAGEBUS(-4), - SEVERAL_CLUSTERS(-5), - URL_PARSING(-6), - INVALID_CREATE_VALUE(-7), - TOO_MANY_PARALLEL_REQUESTS(-8), - MISSING_CLUSTER(-9), INTERNAL_EXCEPTION(-9), - DOCUMENT_CONDITION_NOT_MET(-10), - DOCUMENT_EXCEPTION(-11), - PARSER_ERROR(-11), - GROUP_AND_EXPRESSION_ERROR(-12), - TIME_OUT(-13), - INTERRUPTED(-14), - UNSPECIFIED(-15), - UNKNOWN_BUCKET_SPACE(-16); - - public final long value; - apiErrorCodes(long value) { - this.value = value; - } - } - - /** - * Represents the "grouping" part of document id which can be used with streaming model. - */ - public static class Group { - public final char name; - public final String value; - Group(char name, String value) { - this.name = name; - this.value = value; - } - } - private final String namespace; - private final String documentType; - private final String docId; - private Optional<Group> group = Optional.empty(); - private final String rawPath; - - public boolean isRootOnly() { - return namespace == null; - } - - public String getRawPath() { - return rawPath; - } - - public String getNamespace() { - return namespace; - } - - public String getDocumentType() { - return documentType; - } - - public String getDocId() { - return docId; - } - - public Optional<Group> getGroup() { - return group; - } - - public String generateFullId() { - return ID + namespace + ":" + documentType + ":" - + group.map(g -> String.format("%s=%s", g.name, g.value)).orElse("") - + ":" + docId; - } - - static class PathParser { - public static final long ERROR_ID_DECODING_PATH = -10L; - final List<String> rawParts; - final String originalPath; - int readPos = 0; - public PathParser(String path) { - this.originalPath = path; - this.rawParts = Splitter.on('/').splitToList(path); - } - - boolean hasNextToken() { - return readPos < rawParts.size(); - } - - String nextTokenOrException() throws RestApiException { - if (readPos >= rawParts.size()) { - throwUsage(originalPath); - } - String nextToken = rawParts.get(readPos++); - return urlDecodeOrException(nextToken); - } - - String restOfPath() throws RestApiException { - String rawId = Joiner.on("/").join(rawParts.listIterator(readPos)); - return urlDecodeOrException(rawId); - } - - String urlDecodeOrException(String url) throws RestApiException { - try { - return URLDecoder.decode(url, StandardCharsets.UTF_8.name()); - } catch (UnsupportedEncodingException e) { - throw new RestApiException(Response.createErrorResponse(BAD_REQUEST,"Problems decoding the URI: " + e.getMessage(), apiErrorCodes.ERROR_ID_DECODING_PATH)); - } - } - } - - public RestUri(URI uri) throws RestApiException { - rawPath = uri.getRawPath(); - PathParser pathParser = new PathParser(rawPath); - if (! pathParser.nextTokenOrException().equals("") || - ! pathParser.nextTokenOrException().equals(DOCUMENT) || - ! pathParser.nextTokenOrException().equals(V_1)) { - throwUsage(uri.getRawPath()); - } - // If /document/v1 root request, there's an empty token at the end. - String maybeNamespace = pathParser.nextTokenOrException(); - if (maybeNamespace.isEmpty()) { - namespace = null; - documentType = null; - docId = null; - return; - } - namespace = maybeNamespace; - documentType = pathParser.nextTokenOrException(); - switch (pathParser.nextTokenOrException()) { - case "number": - group = Optional.of(new Group(NUMBER_STREAMING, pathParser.nextTokenOrException())); - break; - case "docid": - group = Optional.empty(); - break; - case "group": - group = Optional.of(new Group(GROUP_STREAMING, pathParser.nextTokenOrException())); - break; - default: throwUsage(uri.getRawPath()); - } - docId = pathParser.restOfPath(); - } - - private static void throwUsage(String inputPath) throws RestApiException { - throw new RestApiException(Response.createErrorResponse(BAD_REQUEST, - "Expected: " + - ".../{namespace}/{document-type}/group/{name}/[{user-specified}] " + - ".../{namespace}/{document-type}/docid/[{user-specified}] : but got " + inputPath, apiErrorCodes.ERROR_ID_BASIC_USAGE)); - } - -} - diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index e603a150b34..bd63a2ecbfc 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -1,443 +1,26 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.restapi.resource; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.inject.Inject; -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.container.handler.ThreadpoolConfig; -import com.yahoo.container.handler.threadpool.ContainerThreadPool; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.container.logging.AccessLog; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.TestAndSetCondition; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.document.json.SingleDocumentParser; -import com.yahoo.document.restapi.OperationHandler; -import com.yahoo.document.restapi.OperationHandlerImpl; -import com.yahoo.document.restapi.Response; -import com.yahoo.document.restapi.RestApiException; -import com.yahoo.document.restapi.RestUri; -import com.yahoo.document.select.DocumentSelector; -import com.yahoo.document.select.parser.ParseException; -import com.yahoo.documentapi.DocumentAccess; -import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; -import com.yahoo.documentapi.messagebus.MessageBusParams; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -import com.yahoo.jdisc.Metric; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.text.Text; -import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.vespaclient.ClusterDef; -import com.yahoo.vespaclient.ClusterList; -import com.yahoo.vespaxmlparser.DocumentFeedOperation; -import com.yahoo.vespaxmlparser.FeedOperation; -import com.yahoo.yolean.Exceptions; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; - -import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; /** - * API for handling single operation on a document and visiting. + * Dummy for internal use. * - * @author Haakon Dybdahl + * @author jonmv */ public class RestApi extends LoggingRequestHandler { - private static final String CREATE_PARAMETER_NAME = "create"; - private static final String CONDITION_PARAMETER_NAME = "condition"; - private static final String ROUTE_PARAMETER_NAME = "route"; - private static final String DOCUMENTS = "documents"; - private static final String FIELDS = "fields"; - private static final String DOC_ID_NAME = "id"; - private static final String PATH_NAME = "pathId"; - private static final String SELECTION = "selection"; - private static final String CLUSTER = "cluster"; - private static final String CONTINUATION = "continuation"; - private static final String WANTED_DOCUMENT_COUNT = "wantedDocumentCount"; - private static final String FIELD_SET = "fieldSet"; - private static final String CONCURRENCY = "concurrency"; - private static final String BUCKET_SPACE = "bucketSpace"; - private static final String APPLICATION_JSON = "application/json"; - private final OperationHandler operationHandler; - private SingleDocumentParser singleDocumentParser; - private final ObjectMapper mapper = new ObjectMapper(); - private final AtomicInteger threadsAvailableForApi; - @Inject - public RestApi(ContainerThreadPool threadpool, - AccessLog accessLog, - Metric metric, - DocumentmanagerConfig documentManagerConfig, - LoadTypeConfig loadTypeConfig, - ThreadpoolConfig threadpoolConfig, - AllClustersBucketSpacesConfig bucketSpacesConfig, - ClusterListConfig clusterListConfig, - MetricReceiver metricReceiver) { - super(threadpool.executor(), accessLog, metric); - MessageBusParams params = new MessageBusParams(new LoadTypeSet(loadTypeConfig)); - params.setDocumentmanagerConfig(documentManagerConfig); - this.operationHandler = new OperationHandlerImpl(new MessageBusDocumentAccess(params), - fixedClusterEnumeratorFromConfig(clusterListConfig), - fixedBucketSpaceResolverFromConfig(bucketSpacesConfig), - metricReceiver); - this.singleDocumentParser = new SingleDocumentParser(new DocumentTypeManager(documentManagerConfig)); - // 40% of the threads can be blocked before we deny requests. - if (threadpoolConfig != null) { - threadsAvailableForApi = new AtomicInteger(Math.max((int) (0.4 * threadpoolConfig.maxthreads()), 1)); - } else { - log.warning("No config for threadpool, using 200 for max blocking threads for document rest API."); - threadsAvailableForApi = new AtomicInteger(200); - } - } - - // For testing and development - RestApi(Executor executor, AccessLog accessLog, OperationHandler operationHandler, int threadsAvailable) { - super(executor, accessLog, null); - this.operationHandler = operationHandler; - this.threadsAvailableForApi = new AtomicInteger(threadsAvailable); - } - - @Override - public void destroy() { - operationHandler.shutdown(); - } - - // For testing and development - protected void setDocTypeManagerForTests(DocumentTypeManager docTypeManager) { - this.singleDocumentParser = new SingleDocumentParser(docTypeManager); - } - - private static OperationHandlerImpl.ClusterEnumerator fixedClusterEnumeratorFromConfig(ClusterListConfig config) { - List<ClusterDef> clusters = Collections.unmodifiableList(new ClusterList(config).getStorageClusters()); - return () -> clusters; - } - - private static OperationHandlerImpl.BucketSpaceResolver fixedBucketSpaceResolverFromConfig(AllClustersBucketSpacesConfig bucketSpacesConfig) { - return (clusterId, docType) -> - Optional.ofNullable(bucketSpacesConfig.cluster(clusterId)) - .map(cluster -> cluster.documentType(docType)) - .map(type -> type.bucketSpace()); - } - - private static Optional<String> requestProperty(String parameter, HttpRequest request) { - String property = request.getProperty(parameter); - if (property != null && ! property.isEmpty()) { - return Optional.of(property); - } - return Optional.empty(); - } - - private static boolean parseBooleanStrict(String value) { - if ("true".equalsIgnoreCase(value)) { - return true; - } else if ("false".equalsIgnoreCase(value)) { - return false; - } - throw new IllegalArgumentException(String.format("Value not convertible to bool: '%s'", value)); - } - - private static Optional<Boolean> parseBoolean(String parameter, HttpRequest request) { - try { - Optional<String> property = requestProperty(parameter, request); - return property.map(RestApi::parseBooleanStrict); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Invalid value for '" + parameter + "' parameter: " + - "Must be empty, true, or false but was '" + - request.getProperty(parameter) + "'"); - } - } - - private static int parsePositiveInt(String str) throws NumberFormatException { - int parsed = Integer.parseInt(str); - if (parsed <= 0) { - throw new IllegalArgumentException("Parsed number was negative or zero"); - } - return parsed; + public RestApi() { + super(ignored -> { throw new IllegalStateException("Not supposed to handle anything"); }, null, null); } @Override public HttpResponse handle(HttpRequest request) { - try { - if (threadsAvailableForApi.decrementAndGet() < 1) { - return Response.createErrorResponse(429 /* Too Many Requests */, - "Too many parallel requests, consider using http-vespa-java-client. Please try again later.", - RestUri.apiErrorCodes.TOO_MANY_PARALLEL_REQUESTS); - } - return handleInternal(request); - } finally { - threadsAvailableForApi.incrementAndGet(); - } - } - - private static void validateUriStructureForRequestMethod(RestUri uri, com.yahoo.jdisc.http.HttpRequest.Method method) - throws RestApiException { - if ((method != com.yahoo.jdisc.http.HttpRequest.Method.GET) && uri.isRootOnly()) { - throw new RestApiException(Response.createErrorResponse(BAD_REQUEST, - "Root /document/v1/ requests only supported for HTTP GET", - RestUri.apiErrorCodes.ERROR_ID_BASIC_USAGE)); - } - } - - private static boolean isVisitRequestUri(RestUri uri) { - return (uri.isRootOnly() || uri.getDocId().isEmpty()); - } - - // protected for testing - protected HttpResponse handleInternal(HttpRequest request) { - RestUri restUri = null; - try { - restUri = new RestUri(request.getUri()); - validateUriStructureForRequestMethod(restUri, request.getMethod()); - - Optional<Boolean> create; - try { - create = parseBoolean(CREATE_PARAMETER_NAME, request); - } - catch (IllegalArgumentException e) { - return Response.createErrorResponse(400, e.getMessage(), RestUri.apiErrorCodes.INVALID_CREATE_VALUE); - } - - String condition = request.getProperty(CONDITION_PARAMETER_NAME); - Optional<String> route = Optional.ofNullable(nonEmpty(request.getProperty(ROUTE_PARAMETER_NAME), ROUTE_PARAMETER_NAME)); - - Optional<ObjectNode> resultJson = Optional.empty(); - switch (request.getMethod()) { - case GET: // Vespa Visit/Get - return isVisitRequestUri(restUri) ? handleVisit(restUri, request) : handleGet(restUri, request); - case POST: // Vespa Put - operationHandler.put(restUri, createPutOperation(request, restUri.generateFullId(), condition), route); - break; - case PUT: // Vespa Update - operationHandler.update(restUri, createUpdateOperation(request, restUri.generateFullId(), condition, create), route); - break; - case DELETE: // Vespa Delete - operationHandler.delete(restUri, condition, route); - break; - default: - return new Response(405, Optional.empty(), Optional.of(restUri)); - } - return new Response(200, resultJson, Optional.of(restUri)); - } - catch (RestApiException e) { - return e.getResponse(); - } - catch (IllegalArgumentException userException) { - return Response.createErrorResponse(400, Exceptions.toMessageString(userException), - restUri, - RestUri.apiErrorCodes.PARSER_ERROR); - } - catch (RuntimeException systemException) { - log.log(Level.WARNING, "Internal runtime exception during Document V1 request handling", systemException); - return Response.createErrorResponse(500, Exceptions.toMessageString(systemException), - restUri, - RestUri.apiErrorCodes.UNSPECIFIED); - } - } - - private FeedOperation createPutOperation(HttpRequest request, String id, String condition) { - FeedOperation put = singleDocumentParser.parsePut(request.getData(), id); - if (condition != null && ! condition.isEmpty()) { - return new DocumentFeedOperation(put.getDocument(), new TestAndSetCondition(condition)); - } - return put; + throw new IllegalStateException("Not supposed to handle anything"); } - private FeedOperation createUpdateOperation(HttpRequest request, String id, String condition, Optional<Boolean> create) { - FeedOperation update = singleDocumentParser.parseUpdate(request.getData(), id); - if (condition != null && ! condition.isEmpty()) { - update.getDocumentUpdate().setCondition(new TestAndSetCondition(condition)); - } - create.ifPresent(c -> update.getDocumentUpdate().setCreateIfNonExistent(c)); - return update; - } - - private HttpResponse handleGet(RestUri restUri, HttpRequest request) throws RestApiException { - final Optional<String> fieldSet = requestProperty(FIELD_SET, request); - final Optional<String> cluster = requestProperty(CLUSTER, request); - final Optional<String> getDocument = operationHandler.get(restUri, fieldSet, cluster); - final ObjectNode resultNode = mapper.createObjectNode(); - if (getDocument.isPresent()) { - final JsonNode parseNode; - try { - parseNode = mapper.readTree(getDocument.get()); - } catch (IOException e) { - throw new RuntimeException("Failed while parsing my own results", e); - } - resultNode.putPOJO(FIELDS, parseNode.get(FIELDS)); - } - resultNode.put(DOC_ID_NAME, restUri.generateFullId()); - resultNode.put(PATH_NAME, restUri.getRawPath()); - - return new HttpResponse(getDocument.isPresent() ? 200 : 404) { - @Override - public String getContentType() { return APPLICATION_JSON; } - @Override - public void render(OutputStream outputStream) throws IOException { - outputStream.write(resultNode.toString().getBytes(StandardCharsets.UTF_8.name())); - } - }; - } - - private static HttpResponse createInvalidParameterResponse(String parameter, String explanation) { - return Response.createErrorResponse(400, String.format("Invalid '%s' value. %s", parameter, explanation), RestUri.apiErrorCodes.UNSPECIFIED); - } - - static class BadRequestParameterException extends IllegalArgumentException { - private String parameter; - - BadRequestParameterException(String parameter, String message) { - super(message); - this.parameter = parameter; - } - - String getParameter() { - return parameter; - } - } - - private static Optional<Integer> parsePositiveIntegerRequestParameter(String parameter, HttpRequest request) { - Optional<String> property = requestProperty(parameter, request); - if (!property.isPresent()) { - return Optional.empty(); - } - try { - return property.map(RestApi::parsePositiveInt); - } catch (IllegalArgumentException e) { - throw new BadRequestParameterException(parameter, "Expected positive integer"); - } - } - - private static OperationHandler.VisitOptions visitOptionsFromRequest(HttpRequest request) { - final OperationHandler.VisitOptions.Builder optionsBuilder = OperationHandler.VisitOptions.builder(); - - Optional.ofNullable(request.getProperty(CLUSTER)).ifPresent(c -> optionsBuilder.cluster(c)); - Optional.ofNullable(request.getProperty(CONTINUATION)).ifPresent(c -> optionsBuilder.continuation(c)); - Optional.ofNullable(request.getProperty(FIELD_SET)).ifPresent(fs -> optionsBuilder.fieldSet(fs)); - Optional.ofNullable(request.getProperty(BUCKET_SPACE)).ifPresent(s -> optionsBuilder.bucketSpace(s)); - parsePositiveIntegerRequestParameter(WANTED_DOCUMENT_COUNT, request).ifPresent(c -> optionsBuilder.wantedDocumentCount(c)); - parsePositiveIntegerRequestParameter(CONCURRENCY, request).ifPresent(c -> optionsBuilder.concurrency(c)); - - return optionsBuilder.build(); - } - - /** - * Escapes all single quotes in input string. - * @param original non-escaped string that may contain single quotes - * @return original if no quotes to escaped were found, otherwise a quote-escaped string - */ - private static String singleQuoteEscapedString(String original) { - if (original.indexOf('\'') == -1) { - return original; - } - StringBuilder builder = new StringBuilder(original.length() + 1); - for (int i = 0; i < original.length(); ++i) { - char c = original.charAt(i); - if (c != '\'') { - builder.append(c); - } else { - builder.append("\\'"); - } - } - return builder.toString(); - } - - - private String nonEmpty(String value, String name) { - if (value != null && value.isEmpty()) - throw new IllegalArgumentException("'" + name + "' cannot be empty"); - return value; - } - - private static long parseAndValidateVisitNumericId(String value) { - try { - return Long.parseLong(value); - } catch (NumberFormatException e) { - throw new BadRequestParameterException(SELECTION, "Failed to parse numeric part of selection URI"); - } - } - - private static String validateAndBuildLocationSubExpression(RestUri.Group group) { - if (group.name == 'n') { - return String.format("id.user==%d", parseAndValidateVisitNumericId(group.value)); - } else { - // Cannot feed documents with groups that don't pass this test, so it makes sense - // to enforce this symmetry when trying to retrieve them as well. - Text.validateTextString(group.value).ifPresent(codepoint -> { - throw new BadRequestParameterException(SELECTION, String.format( - "Failed to parse group part of selection URI; contains invalid text code point U%04X", codepoint)); - }); - return String.format("id.group=='%s'", singleQuoteEscapedString(group.value)); - } - } - - private static void validateDocumentSelectionSyntax(String expression) { - try { - new DocumentSelector(expression); - } catch (ParseException e) { - throw new BadRequestParameterException(SELECTION, String.format("Failed to parse expression given in 'selection'" + - " parameter. Must be a complete and valid sub-expression. Error: %s", e.getMessage())); - } - } - - private static String documentSelectionFromRequest(RestUri restUri, HttpRequest request) throws BadRequestParameterException { - String documentSelection = Optional.ofNullable(request.getProperty(SELECTION)).orElse(""); - if (!documentSelection.isEmpty()) { - // Ensure that the selection parameter sub-expression is complete and valid by itself. - validateDocumentSelectionSyntax(documentSelection); - } - if (restUri.getGroup().isPresent() && ! restUri.getGroup().get().value.isEmpty()) { - String locationSubExpression = validateAndBuildLocationSubExpression(restUri.getGroup().get()); - if (documentSelection.isEmpty()) { - documentSelection = locationSubExpression; - } else { - documentSelection = String.format("%s and (%s)", locationSubExpression, documentSelection); - } - } - return documentSelection; - } - - private HttpResponse handleVisit(RestUri restUri, HttpRequest request) throws RestApiException { - String documentSelection; - OperationHandler.VisitOptions options; - try { - documentSelection = documentSelectionFromRequest(restUri, request); - options = visitOptionsFromRequest(request); - } catch (BadRequestParameterException e) { - return createInvalidParameterResponse(e.getParameter(), e.getMessage()); - } - OperationHandler.VisitResult visit = operationHandler.visit(restUri, documentSelection, options); - ObjectNode resultNode = mapper.createObjectNode(); - visit.token.ifPresent(t -> resultNode.put(CONTINUATION, t)); - resultNode.putArray(DOCUMENTS).addPOJO(visit.documentsAsJsonList); - resultNode.put(PATH_NAME, restUri.getRawPath()); - - HttpResponse httpResponse = new HttpResponse(200) { - @Override - public String getContentType() { return APPLICATION_JSON; } - @Override - public void render(OutputStream outputStream) throws IOException { - try { - outputStream.write(resultNode.toString().getBytes(StandardCharsets.UTF_8)); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - }; - return httpResponse; - } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/documentapi/metrics/DocumentOperationStatus.java b/vespaclient-container-plugin/src/main/java/com/yahoo/documentapi/metrics/DocumentOperationStatus.java index f0529f3d55a..c665eca1cac 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/documentapi/metrics/DocumentOperationStatus.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/documentapi/metrics/DocumentOperationStatus.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.metrics; -import com.yahoo.document.restapi.OperationHandlerImpl; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import java.util.Set; @@ -29,7 +29,10 @@ public enum DocumentOperationStatus { } public static DocumentOperationStatus fromMessageBusErrorCodes(Set<Integer> errorCodes) { - return fromHttpStatusCode(OperationHandlerImpl.getHTTPStatusCode(errorCodes)); + if (errorCodes.size() == 1 && errorCodes.contains(DocumentProtocol.ERROR_NO_SPACE)) + return SERVER_ERROR; + + return REQUEST_ERROR; } } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java deleted file mode 100644 index fd45a0d5dd7..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.restapi; - -import com.yahoo.application.Application; -import com.yahoo.application.Networking; -import org.junit.Test; - -/** - * @author bratseth - */ -public class DocumentApiApplicationTest { - - /** Test that it is possible to instantiate an Application with a document-api */ - @Test - public void application_with_document_api() { - String services = - "<container version='1.0'>" + - " <http><server port=\"0\" id=\"foobar\"/></http>" + - " <document-api/>" + - "</container>"; - try (Application application = Application.fromServicesXml(services, Networking.enable)) { - } - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java deleted file mode 100644 index efb25f0e2b3..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java +++ /dev/null @@ -1,445 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.restapi; - -import com.yahoo.document.fieldset.AllFields; -import com.yahoo.documentapi.DocumentAccess; -import com.yahoo.documentapi.ProgressToken; -import com.yahoo.documentapi.SyncParameters; -import com.yahoo.documentapi.VisitorControlHandler; -import com.yahoo.documentapi.VisitorParameters; -import com.yahoo.documentapi.VisitorSession; -import com.yahoo.documentapi.messagebus.MessageBusSyncSession; -import com.yahoo.messagebus.StaticThrottlePolicy; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.vdslib.VisitorStatistics; -import com.yahoo.vespaclient.ClusterDef; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicReference; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - - -public class OperationHandlerImplTest { - - @Test(expected = IllegalArgumentException.class) - public void missingClusterDef() throws RestApiException { - List<ClusterDef> clusterDef = new ArrayList<>(); - OperationHandlerImpl.resolveClusterDef(Optional.empty(), clusterDef); - } - - @Test(expected = IllegalArgumentException.class) - public void missingClusterDefSpecifiedCluster() throws RestApiException { - List<ClusterDef> clusterDef = new ArrayList<>(); - OperationHandlerImpl.resolveClusterDef(Optional.of("cluster"), clusterDef); - } - - @Test(expected = RestApiException.class) - public void oneClusterPresentNotMatching() throws RestApiException { - List<ClusterDef> clusterDef = new ArrayList<>(); - clusterDef.add(new ClusterDef("foo", "configId")); - OperationHandlerImpl.resolveClusterDef(Optional.of("cluster"), clusterDef); - } - - private static String toRoute(ClusterDef clusterDef) { - return OperationHandlerImpl.clusterDefToRoute(clusterDef); - } - - @Test() - public void oneClusterMatching() throws RestApiException { - List<ClusterDef> clusterDef = new ArrayList<>(); - clusterDef.add(new ClusterDef("foo", "configId")); - assertThat(toRoute(OperationHandlerImpl.resolveClusterDef(Optional.of("foo"), clusterDef)), - is("[Storage:cluster=foo;clusterconfigid=configId]")); - } - - @Test() - public void oneClusterMatchingManyAvailable() throws RestApiException { - List<ClusterDef> clusterDef = new ArrayList<>(); - clusterDef.add(new ClusterDef("foo2", "configId2")); - clusterDef.add(new ClusterDef("foo", "configId")); - clusterDef.add(new ClusterDef("foo3", "configId2")); - assertThat(toRoute(OperationHandlerImpl.resolveClusterDef(Optional.of("foo"), clusterDef)), - is("[Storage:cluster=foo;clusterconfigid=configId]")); - } - - @Test() - public void unknown_target_cluster_throws_exception() throws RestApiException, IOException { - List<ClusterDef> clusterDef = new ArrayList<>(); - clusterDef.add(new ClusterDef("foo2", "configId2")); - clusterDef.add(new ClusterDef("foo", "configId")); - clusterDef.add(new ClusterDef("foo3", "configId2")); - try { - OperationHandlerImpl.resolveClusterDef(Optional.of("wrong"), clusterDef); - } catch(RestApiException e) { - assertThat(e.getResponse().getStatus(), is(400)); - String errorMsg = renderRestApiExceptionAsString(e); - assertThat(errorMsg, is("{\"errors\":[{\"description\":" + - "\"MISSING_CLUSTER Your vespa cluster contains the content clusters 'foo2', 'foo'," + - " 'foo3', not 'wrong'. Please select a valid vespa cluster.\",\"id\":-9}]}")); - return; - } - fail("Expected exception"); - } - - private String renderRestApiExceptionAsString(RestApiException e) throws IOException { - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - e.getResponse().render(stream); - return new String( stream.toByteArray()); - } - - private static class OperationHandlerImplFixture { - DocumentAccess documentAccess = mock(DocumentAccess.class); - AtomicReference<VisitorParameters> assignedParameters = new AtomicReference<>(); - VisitorControlHandler.CompletionCode completionCode = VisitorControlHandler.CompletionCode.SUCCESS; - int bucketsVisited = 0; - Map<String, String> bucketSpaces = new HashMap<>(); - MessageBusSyncSession mockSyncSession = mock(MessageBusSyncSession.class); // MBus session needed to avoid setRoute throwing. - - OperationHandlerImplFixture() { - bucketSpaces.put("foo", "global"); - bucketSpaces.put("document-type", "default"); - } - - OperationHandlerImpl createHandler() throws Exception { - VisitorSession visitorSession = mock(VisitorSession.class); - // Pre-bake an already completed session - when(documentAccess.createVisitorSession(any(VisitorParameters.class))).thenAnswer(p -> { - VisitorParameters params = (VisitorParameters)p.getArguments()[0]; - assignedParameters.set(params); - - VisitorStatistics statistics = new VisitorStatistics(); - statistics.setBucketsVisited(bucketsVisited); - params.getControlHandler().onVisitorStatistics(statistics); - - ProgressToken progress = new ProgressToken(); - params.getControlHandler().onProgress(progress); - - params.getControlHandler().onDone(completionCode, "bork bork"); - return visitorSession; - }); - when(documentAccess.createSyncSession(any(SyncParameters.class))).thenReturn(mockSyncSession); - OperationHandlerImpl.ClusterEnumerator clusterEnumerator = () -> Arrays.asList(new ClusterDef("foo", "configId")); - OperationHandlerImpl.BucketSpaceResolver bucketSpaceResolver = (clusterId, docType) -> Optional.ofNullable(bucketSpaces.get(docType)); - return new OperationHandlerImpl(documentAccess, clusterEnumerator, bucketSpaceResolver, MetricReceiver.nullImplementation); - } - } - - private static OperationHandler.VisitOptions.Builder optionsBuilder() { - return OperationHandler.VisitOptions.builder(); - } - - private static RestUri dummyVisitUri() throws Exception { - return new RestUri(new URI("http://localhost/document/v1/namespace/document-type/docid/")); - } - - private static RestUri apiRootVisitUri() throws Exception { - return new RestUri(new URI("http://localhost/document/v1/")); - } - - private static RestUri dummyGetUri() throws Exception { - return new RestUri(new URI("http://localhost/document/v1/namespace/document-type/docid/foo")); - } - - private static OperationHandler.VisitOptions visitOptionsWithWantedDocumentCount(int wantedDocumentCount) { - return optionsBuilder().wantedDocumentCount(wantedDocumentCount).build(); - } - - private static OperationHandler.VisitOptions emptyVisitOptions() { - return optionsBuilder().build(); - } - - @Test - public void timeout_without_buckets_visited_throws_timeout_error() throws Exception { - OperationHandlerImplFixture fixture = new OperationHandlerImplFixture(); - fixture.completionCode = VisitorControlHandler.CompletionCode.TIMEOUT; - fixture.bucketsVisited = 0; - // RestApiException hides its guts internally, so cannot trivially use @Rule directly to check for error category - try { - OperationHandlerImpl handler = fixture.createHandler(); - handler.visit(dummyVisitUri(), "", emptyVisitOptions()); - fail("Exception expected"); - } catch (RestApiException e) { - assertThat(e.getResponse().getStatus(), is(500)); - assertThat(renderRestApiExceptionAsString(e), containsString("Timed out")); - } - } - - @Test - public void timeout_with_buckets_visited_does_not_throw_timeout_error() throws Exception { - OperationHandlerImplFixture fixture = new OperationHandlerImplFixture(); - fixture.completionCode = VisitorControlHandler.CompletionCode.TIMEOUT; - fixture.bucketsVisited = 1; - - OperationHandlerImpl handler = fixture.createHandler(); - handler.visit(dummyVisitUri(), "", emptyVisitOptions()); - } - - @Test - public void handler_sets_default_visitor_session_timeout_parameter() throws Exception { - OperationHandlerImplFixture fixture = new OperationHandlerImplFixture(); - OperationHandlerImpl handler = fixture.createHandler(); - - handler.visit(dummyVisitUri(), "", emptyVisitOptions()); - - assertThat(fixture.assignedParameters.get().getSessionTimeoutMs(), is((long)OperationHandlerImpl.VISIT_TIMEOUT_MS)); - } - - private static VisitorParameters generatedVisitParametersFrom(RestUri restUri, String documentSelection, - OperationHandler.VisitOptions options) throws Exception { - OperationHandlerImplFixture fixture = new OperationHandlerImplFixture(); - OperationHandlerImpl handler = fixture.createHandler(); - - handler.visit(restUri, documentSelection, options); - return fixture.assignedParameters.get(); - } - - private static VisitorParameters generatedParametersFromVisitOptions(OperationHandler.VisitOptions options) throws Exception { - return generatedVisitParametersFrom(dummyVisitUri(), "", options); - } - - @Test - public void document_type_is_mapped_to_correct_bucket_space() throws Exception { - OperationHandlerImplFixture fixture = new OperationHandlerImplFixture(); - fixture.bucketSpaces.put("document-type", "langbein"); - OperationHandlerImpl handler = fixture.createHandler(); - handler.visit(dummyVisitUri(), "", emptyVisitOptions()); - - VisitorParameters parameters = fixture.assignedParameters.get(); - assertEquals("langbein", parameters.getBucketSpace()); - } - - @Test - public void unknown_bucket_space_mapping_throws_exception() throws Exception { - OperationHandlerImplFixture fixture = new OperationHandlerImplFixture(); - fixture.bucketSpaces.remove("document-type"); - try { - OperationHandlerImpl handler = fixture.createHandler(); - handler.visit(dummyVisitUri(), "", emptyVisitOptions()); - fail("Exception expected"); - } catch (RestApiException e) { - assertThat(e.getResponse().getStatus(), is(400)); - String errorMsg = renderRestApiExceptionAsString(e); - // FIXME isn't this really more of a case of unknown document type..? - assertThat(errorMsg, is("{\"errors\":[{\"description\":" + - "\"UNKNOWN_BUCKET_SPACE Document type 'document-type' in cluster 'foo' is not mapped to a known bucket space\",\"id\":-16}]}")); - } - } - - @Test - public void provided_wanted_document_count_is_propagated_to_visitor_parameters() throws Exception { - VisitorParameters params = generatedParametersFromVisitOptions(visitOptionsWithWantedDocumentCount(123)); - assertThat(params.getMaxTotalHits(), is((long)123)); - } - - @Test - public void wanted_document_count_is_1_unless_specified() throws Exception { - VisitorParameters params = generatedParametersFromVisitOptions(emptyVisitOptions()); - assertThat(params.getMaxTotalHits(), is((long)1)); - } - - @Test - public void too_low_wanted_document_count_is_bounded_to_1() throws Exception { - VisitorParameters params = generatedParametersFromVisitOptions(visitOptionsWithWantedDocumentCount(-1)); - assertThat(params.getMaxTotalHits(), is((long)1)); - - params = generatedParametersFromVisitOptions(visitOptionsWithWantedDocumentCount(Integer.MIN_VALUE)); - assertThat(params.getMaxTotalHits(), is((long)1)); - - params = generatedParametersFromVisitOptions(visitOptionsWithWantedDocumentCount(0)); - assertThat(params.getMaxTotalHits(), is((long)1)); - } - - @Test - public void too_high_wanted_document_count_is_bounded_to_upper_bound() throws Exception { - VisitorParameters params = generatedParametersFromVisitOptions(visitOptionsWithWantedDocumentCount(OperationHandlerImpl.WANTED_DOCUMENT_COUNT_UPPER_BOUND + 1)); - assertThat(params.getMaxTotalHits(), is((long)OperationHandlerImpl.WANTED_DOCUMENT_COUNT_UPPER_BOUND)); - - params = generatedParametersFromVisitOptions(visitOptionsWithWantedDocumentCount(Integer.MAX_VALUE)); - assertThat(params.getMaxTotalHits(), is((long)OperationHandlerImpl.WANTED_DOCUMENT_COUNT_UPPER_BOUND)); - } - - @Test - public void visit_field_set_covers_all_fields_by_default() throws Exception { - VisitorParameters params = generatedParametersFromVisitOptions(emptyVisitOptions()); - assertThat(params.fieldSet(), equalTo("document-type:[document]")); - } - - @Test - public void provided_visit_fieldset_is_propagated_to_visitor_parameters() throws Exception { - VisitorParameters params = generatedParametersFromVisitOptions(optionsBuilder().fieldSet("document-type:bjarne").build()); - assertThat(params.fieldSet(), equalTo("document-type:bjarne")); - } - - private void assertConcurrencyPropagated(VisitorParameters params, int expectedConcurrency) { - assertThat(params.getThrottlePolicy(), instanceOf(StaticThrottlePolicy.class)); - assertThat(((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount(), is(expectedConcurrency)); - } - - @Test - public void visit_concurrency_is_1_by_default() throws Exception { - VisitorParameters params = generatedParametersFromVisitOptions(emptyVisitOptions()); - assertConcurrencyPropagated(params, 1); - } - - @Test - public void visit_concurrency_is_propagated_to_visitor_parameters() throws Exception { - VisitorParameters params = generatedParametersFromVisitOptions(optionsBuilder().concurrency(3).build()); - assertConcurrencyPropagated(params, 3); - } - - @Test - public void too_low_visit_concurrency_is_capped_to_1() throws Exception { - VisitorParameters params = generatedParametersFromVisitOptions(optionsBuilder().concurrency(0).build()); - assertConcurrencyPropagated(params, 1); - } - - @Test - public void too_high_visit_concurrency_is_capped_to_max() throws Exception { - VisitorParameters params = generatedParametersFromVisitOptions( - optionsBuilder().concurrency(OperationHandlerImpl.CONCURRENCY_UPPER_BOUND + 1).build()); - assertConcurrencyPropagated(params, OperationHandlerImpl.CONCURRENCY_UPPER_BOUND); - } - - @Test - public void get_field_covers_all_fields_by_default() throws Exception { - OperationHandlerImplFixture fixture = new OperationHandlerImplFixture(); - OperationHandlerImpl handler = fixture.createHandler(); - handler.get(dummyGetUri(), Optional.empty()); - - verify(fixture.mockSyncSession).get(any(), eq("document-type:[document]"), any()); - } - - @Test - public void provided_get_fieldset_is_propagated_to_sync_session() throws Exception { - OperationHandlerImplFixture fixture = new OperationHandlerImplFixture(); - OperationHandlerImpl handler = fixture.createHandler(); - handler.get(dummyGetUri(), Optional.of("donald,duck")); - - verify(fixture.mockSyncSession).get(any(), eq("donald,duck"), any()); - } - - @Test - public void get_route_has_default_value_if_no_cluster_is_provided() throws Exception { - OperationHandlerImplFixture fixture = new OperationHandlerImplFixture(); - OperationHandlerImpl handler = fixture.createHandler(); - handler.get(dummyGetUri(), Optional.empty(), Optional.empty()); - - // TODO shouldn't this be default-get? - verify(fixture.mockSyncSession).setRoute(eq("default")); - } - - @Test - public void provided_get_cluster_is_propagated_as_route_to_sync_session() throws Exception { - OperationHandlerImplFixture fixture = new OperationHandlerImplFixture(); - OperationHandlerImpl handler = fixture.createHandler(); - handler.get(dummyGetUri(), Optional.empty(), Optional.of("foo")); - - verify(fixture.mockSyncSession).setRoute(eq("[Storage:cluster=foo;clusterconfigid=configId]")); - } - - @Test - public void api_root_visit_uri_requires_cluster_set() throws Exception { - OperationHandlerImplFixture fixture = new OperationHandlerImplFixture(); - OperationHandlerImpl handler = fixture.createHandler(); - try { - handler.visit(apiRootVisitUri(), "", emptyVisitOptions()); - fail("Exception expected"); - } catch (RestApiException e) { - assertThat(e.getResponse().getStatus(), is(400)); - assertThat(renderRestApiExceptionAsString(e), containsString( - "MISSING_CLUSTER Must set 'cluster' parameter to a valid content cluster id " + - "when visiting at a root /document/v1/ level")); - } - } - - @Test - public void api_root_visiting_propagates_request_route() throws Exception { - VisitorParameters parameters = generatedVisitParametersFrom(apiRootVisitUri(), "", optionsBuilder().cluster("foo").build()); - assertEquals("[Storage:cluster=foo;clusterconfigid=configId]", parameters.getRoute().toString()); - } - - @Test - public void api_root_visiting_targets_default_bucket_space_by_default() throws Exception { - VisitorParameters parameters = generatedVisitParametersFrom(apiRootVisitUri(), "", optionsBuilder().cluster("foo").build()); - assertEquals("default", parameters.getBucketSpace()); - } - - @Test - public void api_root_visiting_can_explicitly_specify_bucket_space() throws Exception { - VisitorParameters parameters = generatedVisitParametersFrom(apiRootVisitUri(), "", - optionsBuilder().cluster("foo").bucketSpace("global").build()); - assertEquals("global", parameters.getBucketSpace()); - } - - @Test - public void api_root_visiting_throws_exception_on_unknown_bucket_space_name() throws Exception { - try { - generatedVisitParametersFrom(apiRootVisitUri(), "", optionsBuilder().cluster("foo").bucketSpace("langbein").build()); - } catch (RestApiException e) { - assertThat(e.getResponse().getStatus(), is(400)); - assertThat(renderRestApiExceptionAsString(e), containsString( - "UNKNOWN_BUCKET_SPACE Bucket space 'langbein' is not a known bucket space " + - "(expected 'default' or 'global')")); - } - } - - @Test - public void api_root_visiting_has_empty_document_selection_by_default() throws Exception { - VisitorParameters parameters = generatedVisitParametersFrom(apiRootVisitUri(), "", optionsBuilder().cluster("foo").build()); - assertEquals("", parameters.getDocumentSelection()); - } - - @Test - public void api_root_visiting_propagates_provided_document_selection() throws Exception { - VisitorParameters parameters = generatedVisitParametersFrom(apiRootVisitUri(), "baz.blarg", optionsBuilder().cluster("foo").build()); - // Note: syntax correctness of selection is checked and enforced by RestApi - assertEquals("baz.blarg", parameters.getDocumentSelection()); - } - - @Test - public void api_root_visiting_uses_all_fieldset_by_default() throws Exception { - VisitorParameters parameters = generatedVisitParametersFrom(apiRootVisitUri(), "", optionsBuilder().cluster("foo").build()); - assertEquals(AllFields.NAME, parameters.getFieldSet()); - } - - @Test - public void api_root_visiting_propagates_provided_fieldset() throws Exception { - VisitorParameters parameters = generatedVisitParametersFrom(apiRootVisitUri(), "", - optionsBuilder().cluster("foo").fieldSet("zoidberg:[document]").build()); - assertEquals("zoidberg:[document]", parameters.getFieldSet()); - } - - @Test - public void namespace_and_doctype_augmented_selection_has_parenthesized_selection_sub_expression() throws Exception { - VisitorParameters parameters = generatedVisitParametersFrom(dummyVisitUri(), "1 != 2", optionsBuilder().cluster("foo").build()); - assertEquals("((1 != 2) and document-type and (id.namespace=='namespace'))", parameters.getDocumentSelection()); - } - - @Test - public void namespace_and_doctype_visit_without_selection_does_not_contain_selection_sub_expression() throws Exception { - VisitorParameters parameters = generatedVisitParametersFrom(dummyVisitUri(), "", optionsBuilder().cluster("foo").build()); - assertEquals("document-type and (id.namespace=='namespace')", parameters.getDocumentSelection()); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java deleted file mode 100644 index bdeee12a32a..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.restapi; - -import org.apache.http.client.utils.URIBuilder; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.Optional; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -public class RestUriTest { - - URI createUri(String path, String query) throws URISyntaxException { - return new URIBuilder() - .addParameter("foo", "bar") - .setHost("host") - .setScheme("http") - .setPort(666) - .setPath(path) - .setCustomQuery(query) - .setFragment("fargment").build(); - } - - @Rule - public ExpectedException thrown= ExpectedException.none(); - - @Test - public void testBasic() throws Exception { - RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/myid", "query")); - assertThat(restUri.getDocId(), is("myid")); - assertThat(restUri.getDocumentType(), is("doctype")); - assertThat(restUri.getNamespace(), is("namespace")); - assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty())); - assertThat(restUri.generateFullId(), is("id:namespace:doctype::myid")); - } - - @Test - public void encodingSlashes() throws Exception { - // Try with slashes encoded. - final String id = " !\"øæåp/:;&,.:;'1Q"; - String encodedId = URLEncoder.encode(id, StandardCharsets.UTF_8.name()); - RestUri restUri = new RestUri(URI.create("/document/v1/namespace/doctype/docid/" + encodedId)); - assertThat(restUri.getDocId(), is(id)); - assertThat(restUri.getDocumentType(), is("doctype")); - assertThat(restUri.getNamespace(), is("namespace")); - assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty())); - assertThat(restUri.generateFullId(), is("id:namespace:doctype::" + id)); - } - - @Test - public void encodingSlashes2() throws Exception { - // This will decode the slashes. - final String id = " !\"øæåp/:;&,.:;'1Q "; - RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/" + id, "query")); - assertThat(restUri.getDocId(), is(id)); - assertThat(restUri.getDocumentType(), is("doctype")); - assertThat(restUri.getNamespace(), is("namespace")); - assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty())); - assertThat(restUri.generateFullId(), is("id:namespace:doctype::" + id)); - } - - - @Test - public void testVisit() throws Exception { - RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/", "query")); - assertThat(restUri.getDocId(), is("")); - assertThat(restUri.getDocumentType(), is("doctype")); - assertThat(restUri.getNamespace(), is("namespace")); - assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty())); - assertThat(restUri.generateFullId(), is("id:namespace:doctype::")); - } - - @Test - public void testOneSlashTooMuchWhichIsFine() throws Exception { - RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/myid:342:23/wrong", "")); - assertThat(restUri.getDocId(), is("myid:342:23/wrong")); - } - - @Test - public void testGroupG() throws Exception { - RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/group/group/myid", "")); - assertThat(restUri.getDocId(), is("myid")); - assertThat(restUri.getDocumentType(), is("doctype")); - assertThat(restUri.getGroup().get().name, is('g')); - assertThat(restUri.getGroup().get().value, is("group")); - assertThat(restUri.generateFullId(), is("id:namespace:doctype:g=group:myid")); - } - - @Test - public void testGroupUrlDecode() throws Exception { - RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/group/group#123/myid", "")); - assertThat(restUri.getDocId(), is("myid")); - assertThat(restUri.getDocumentType(), is("doctype")); - assertThat(restUri.getGroup().get().name, is('g')); - assertThat(restUri.getGroup().get().value, is("group#123")); - assertThat(restUri.generateFullId(), is("id:namespace:doctype:g=group#123:myid")); - } - - @Test - public void testGroupN() throws Exception { - RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/number/group/myid", "")); - assertThat(restUri.getGroup().get().name, is('n')); - assertThat(restUri.getGroup().get().value, is("group")); - } - - @Test - public void testGroupUnknown() throws Exception { - thrown.expect(RestApiException.class); - new RestUri(createUri("/document/v1/namespace/doctype/Q/myid", "")); - } - - @Test - public void testDocIdAsIs() throws Exception { - RestUri restUri = new RestUri(new URI("/document/v1/test/newsarticle/docid/http%3a%2f%2fvn.news.yahoo.com%2fgi-th-ng-t-n-ng-khoa-h-205000458.html").normalize()); - assertThat(restUri.getNamespace(), is("test")); - assertThat(restUri.getDocumentType(), is("newsarticle")); - assertThat(restUri.getDocId(), is("http://vn.news.yahoo.com/gi-th-ng-t-n-ng-khoa-h-205000458.html")); - assertThat(restUri.generateFullId(), is("id:test:newsarticle::http://vn.news.yahoo.com/gi-th-ng-t-n-ng-khoa-h-205000458.html")); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/feed-document1.json b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/feed-document1.json deleted file mode 100644 index e69de29bb2d..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/feed-document1.json +++ /dev/null diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java deleted file mode 100644 index eb6bb609970..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.restapi.resource; - -import com.yahoo.document.restapi.OperationHandler; -import com.yahoo.document.restapi.Response; -import com.yahoo.document.restapi.RestApiException; -import com.yahoo.document.restapi.RestUri; -import com.yahoo.vespaxmlparser.FeedOperation; - -import java.util.Optional; - -/** - * Mock that collects info about operation and returns them on second delete. - */ -public class MockedOperationHandler implements OperationHandler { - - StringBuilder log = new StringBuilder(); - int deleteCount = 0; - - @Override - public VisitResult visit(RestUri restUri, String documentSelection, VisitOptions options) throws RestApiException { - return new VisitResult(Optional.of("token"), "List of json docs, cont token " - + options.continuation.orElse("not set") + ", doc selection: '" - + documentSelection + "'" - + options.wantedDocumentCount.map(n -> String.format(", min docs returned: %d", n)).orElse("") - + options.fieldSet.map(s -> String.format(", field set: '%s'", s)).orElse("") - + options.concurrency.map(n -> String.format(", concurrency: %d", n)).orElse("") - + options.bucketSpace.map(s -> String.format(", bucket space: '%s'", s)).orElse("") - + options.cluster.map(s -> String.format(", cluster: '%s'", s)).orElse("")); - } - - @Override - @SuppressWarnings("deprecation") - public void put(RestUri restUri, FeedOperation data, Optional<String> route) throws RestApiException { - log.append("PUT: " + data.getDocument().getId()); - log.append(data.getDocument().getHeader().toString()); - } - - @Override - public void update(RestUri restUri, FeedOperation data, Optional<String> route) throws RestApiException { - log.append("UPDATE: " + data.getDocumentUpdate().getId()); - log.append(data.getDocumentUpdate().fieldUpdates().toString()); - if (data.getDocumentUpdate().getCreateIfNonExistent()) { - log.append("[CREATE IF NON EXISTENT IS TRUE]"); - } - } - - @Override - public void delete(RestUri restUri, String condition, Optional<String> route) throws RestApiException { - deleteCount++; - if (deleteCount == 2) { - String theLog = log.toString(); - log = new StringBuilder(); - deleteCount = 0; - throw new RestApiException(Response.createErrorResponse(666, theLog, RestUri.apiErrorCodes.ERROR_ID_BASIC_USAGE)); - } - log.append("DELETE: " + restUri.generateFullId()); - } - - @Override - public Optional<String> get(RestUri restUri, Optional<String> fieldSet, Optional<String> cluster) throws RestApiException { - log.append("GET: " + restUri.generateFullId()); - // This is _not_ an elegant way to return data back to the test. - // An alternative is removing this entire class in favor of explicit mock expectations. - if (!fieldSet.isPresent() && !cluster.isPresent()) { - return Optional.empty(); - } - return Optional.of(String.format("{\"fields\": {\"fieldset\": \"%s\",\"cluster\":\"%s\"}}", - fieldSet.orElse(""), cluster.orElse(""))); - } - - @Override - public Optional<String> get(RestUri restUri, Optional<String> fieldSet) throws RestApiException { - return get(restUri, fieldSet, Optional.empty()); - } - - @Override - public Optional<String> get(RestUri restUri) throws RestApiException { - return get(restUri, Optional.empty()); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiMaxThreadTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiMaxThreadTest.java deleted file mode 100644 index 39d5617dd4f..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiMaxThreadTest.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.restapi.resource; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.document.restapi.OperationHandler; -import org.junit.Test; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; - -public class RestApiMaxThreadTest { - final CountDownLatch latch = new CountDownLatch(1); - final AtomicInteger requestsInFlight = new AtomicInteger(0); - private class RestApiMocked extends RestApi { - - public RestApiMocked() { - super(mock(Executor.class), null, (OperationHandler)null, 20); - } - - @Override - protected HttpResponse handleInternal(HttpRequest request) { - requestsInFlight.incrementAndGet(); - try { - latch.await(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - return null; - } - } - - @Test - public void testCallsAreThrottled() throws InterruptedException { - RestApiMocked restApiMocked = new RestApiMocked(); - // Fire lots of requests. - for (int x = 0; x < 30; x++) { - new Thread(() -> restApiMocked.handle(null)).start(); - } - // Wait for all threads to be used - while (requestsInFlight.get() != 19) { - Thread.sleep(1); - } - // A new request should be blocked. - final HttpResponse response = restApiMocked.handle(null); - assertThat(response.getStatus(), is(429)); - latch.countDown(); - } -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java deleted file mode 100644 index 0661363477f..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java +++ /dev/null @@ -1,537 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.restapi.resource; - -import com.yahoo.application.Application; -import com.yahoo.application.Networking; -import com.yahoo.application.container.handler.Request; -import com.yahoo.container.Container; -import com.yahoo.jdisc.http.server.jetty.JettyHttpServer; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; -import java.util.function.Function; - -import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsNot.not; -import static org.hamcrest.core.StringContains.containsString; -import static org.hamcrest.core.StringStartsWith.startsWith; -import static org.junit.Assert.assertThat; - -public class RestApiTest { - - Application application; - - @Before - public void setup() throws Exception { - application = Application.fromApplicationPackage(Paths.get("src/test/rest-api-application"), Networking.enable); - } - - @After - public void tearDown() throws Exception { - application.close(); - } - - private static class Response { - final int code; - final String body; - - Response(int code, String body) { - this.code = code; - this.body = body; - } - } - - String post_test_uri = "/document/v1/namespace/testdocument/docid/c"; - String post_test_doc = "{\n" + - "\"foo\" : \"bar\"," + - "\"fields\": {\n" + - "\"title\": \"This is the title\",\n" + - "\"body\": \"This is the body\"" + - "}" + - "}"; - String post_test_response = "{\"id\":\"id:namespace:testdocument::c\"," + - "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; - - // Run this test to manually do request against the REST-API with backend mock. - @Ignore - @Test - public void blockingTest() throws Exception { - System.out.println("Running on port " + getFirstListenPort()); - Thread.sleep(Integer.MAX_VALUE); - } - - @Test - public void testbasicPost() throws Exception { - Request request = new Request("http://localhost:" + getFirstListenPort() + post_test_uri); - HttpPost httpPost = new HttpPost(request.getUri()); - StringEntity entity = new StringEntity(post_test_doc, ContentType.create("application/json")); - httpPost.setEntity(entity); - Response response = doRest(httpPost); - assertThat(response.code, is(200)); - assertThat(response.body, is(post_test_response)); - } - - String post_test_uri_cond = "/document/v1/namespace/testdocument/docid/c?condition=foo"; - String post_test_doc_cond = "{\n" + - "\"foo\" : \"bar\"," + - "\"fields\": {\n" + - "\"title\": \"This is the title\",\n" + - "\"body\": \"This is the body\"" + - "}" + - "}"; - String post_test_response_cond = "{\"id\":\"id:namespace:testdocument::c\"," + - "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; - - @Test - public void testConditionalPost() throws Exception { - Request request = new Request("http://localhost:" + getFirstListenPort() + post_test_uri_cond); - HttpPost httpPost = new HttpPost(request.getUri()); - StringEntity entity = new StringEntity(post_test_doc_cond, ContentType.create("application/json")); - httpPost.setEntity(entity); - Response response = doRest(httpPost); - assertThat(response.code, is(200)); - assertThat(response.body, is(post_test_response_cond)); - } - - @Test - public void testEmptyPost() throws Exception { - Request request = new Request("http://localhost:" + getFirstListenPort() + post_test_uri); - HttpPost httpPost = new HttpPost(request.getUri()); - StringEntity entity = new StringEntity("", ContentType.create("application/json")); - httpPost.setEntity(entity); - assertHttp400ResponseContains(doRest(httpPost), "Could not read document, no document?"); - } - - String update_test_uri = "/document/v1/namespace/testdocument/docid/c"; - String update_test_doc = "{\n" + - "\t\"fields\": {\n" + - "\"title\": {\n" + - "\"assign\": \"Oh lala\"\n" + - "}\n" + - "}\n" + - "}\n"; - - String update_test_response = "{\"id\":\"id:namespace:testdocument::c\"," + - "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; - - @Test - public void testbasicUpdate() throws Exception { - Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_uri); - HttpPut httpPut = new HttpPut(request.getUri()); - StringEntity entity = new StringEntity(update_test_doc, ContentType.create("application/json")); - httpPut.setEntity(entity); - Response response = doRest(httpPut); - assertThat(response.code, is(200)); - assertThat(response.body, is(update_test_response)); - assertThat(getLog(), not(containsString("CREATE IF NON EXISTING IS TRUE"))); - } - - @Test - public void testbasicUpdateCreateTrue() throws Exception { - Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_uri + "?create=true"); - HttpPut httpPut = new HttpPut(request.getUri()); - StringEntity entity = new StringEntity(update_test_doc, ContentType.create("application/json")); - httpPut.setEntity(entity); - Response response = doRest(httpPut); - assertThat(response.code, is(200)); - assertThat(response.body, is(update_test_response)); - assertThat(getLog(), containsString("CREATE IF NON EXISTENT IS TRUE")); - } - - String update_test_create_if_non_existient_uri = "/document/v1/namespace/testdocument/docid/c"; - String update_test_create_if_non_existient_doc = "{\n" + - "\"create\":true," + - "\t\"fields\": {\n" + - "\"title\": {\n" + - "\"assign\": \"Oh lala\"\n" + - "}\n" + - "}\n" + - "}\n"; - - String update_test_create_if_non_existing_response = "{\"id\":\"id:namespace:testdocument::c\"," + - "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; - - @Test - public void testCreateIfNonExistingUpdateInDocTrue() throws Exception { - Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_create_if_non_existient_uri); - HttpPut httpPut = new HttpPut(request.getUri()); - StringEntity entity = new StringEntity(update_test_create_if_non_existient_doc, ContentType.create("application/json")); - httpPut.setEntity(entity); - assertThat(doRest(httpPut).body, is(update_test_create_if_non_existing_response)); - assertThat(getLog(), containsString("CREATE IF NON EXISTENT IS TRUE")); - } - - @Test - public void testCreateIfNonExistingUpdateInDocTrueButQueryParamsFalse() throws Exception { - Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_create_if_non_existient_uri + "?create=false"); - HttpPut httpPut = new HttpPut(request.getUri()); - StringEntity entity = new StringEntity(update_test_create_if_non_existient_doc, ContentType.create("application/json")); - httpPut.setEntity(entity); - assertThat(doRest(httpPut).body, is(update_test_create_if_non_existing_response)); - assertThat(getLog(), not(containsString("CREATE IF NON EXISTENT IS TRUE"))); - } - - @Test - public void bogus_create_parameter_value_returns_http_400_error() throws Exception { - Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_uri + "?create=batman"); - HttpPut httpPut = new HttpPut(request.getUri()); - StringEntity entity = new StringEntity(update_test_doc, ContentType.create("application/json")); - httpPut.setEntity(entity); - assertHttp400ResponseContains(doRest(httpPut), "Invalid value for 'create' parameter: Must be empty, true, or false but was 'batman'"); - } - - // Get logs through some hackish fetch method. Logs is something the mocked backend write. - String getLog() throws IOException { - // The mocked backend will throw a runtime exception with a log if delete is called three times.. - Request request = new Request("http://localhost:" + getFirstListenPort() + remove_test_uri); - HttpDelete delete = new HttpDelete(request.getUri()); - doRest(delete); - return doRest(delete).body; - } - - - String remove_test_uri = "/document/v1/namespace/testdocument/docid/c"; - String remove_test_response = "{\"id\":\"id:namespace:testdocument::c\"," + - "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; - - @Test - public void testbasicRemove() throws Exception { - Request request = new Request("http://localhost:" + getFirstListenPort() + remove_test_uri); - HttpDelete delete = new HttpDelete(request.getUri()); - Response response = doRest(delete); - assertThat(response.code, is(200)); - assertThat(response.body, is(remove_test_response)); - } - - String get_test_uri = "/document/v1/namespace/document-type/docid/c"; - String get_response_part1 = "\"pathId\":\"/document/v1/namespace/document-type/docid/c\""; - String get_response_part2 = "\"id\":\"id:namespace:document-type::c\""; - - - @Test - public void testbasicGet() throws Exception { - Request request = new Request("http://localhost:" + getFirstListenPort() + get_test_uri); - HttpGet get = new HttpGet(request.getUri()); - Response response = doRest(get); - assertThat(response.code, is(404)); // Mock returns Not Found - assertThat(response.body, containsString(get_response_part1)); - assertThat(response.body, containsString(get_response_part2)); - } - - String id_test_uri = "/document/v1/namespace/document-type/docid/f/u/n/n/y/!"; - String id_response_part1 = "\"pathId\":\"/document/v1/namespace/document-type/docid/f/u/n/n/y/!\""; - String id_response_part2 = "\"id\":\"id:namespace:document-type::f/u/n/n/y/!\""; - - @Test - public void testSlashesInId() throws Exception { - Request request = new Request("http://localhost:" + getFirstListenPort() + id_test_uri); - HttpGet get = new HttpGet(request.getUri()); - Response response = doRest(get); - assertThat(response.code, is(404)); // Mock returns Not Found - assertThat(response.body, containsString(id_response_part1)); - assertThat(response.body, containsString(id_response_part2)); - } - - - String get_enc_id = "!\":æøå@/& Q1+"; - // Space encoded as %20, not encoding ! - String get_enc_id_encoded_v1 = "!%22%3A%C3%A6%C3%B8%C3%A5%40%2F%26%20Q1%2B"; - // Space encoded as + - String get_enc_id_encoded_v2 = "%21%22%3A%C3%A6%C3%B8%C3%A5%40%2F%26+Q1%2B"; - String get_enc_test_uri_v1 = "/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v1; - String get_enc_test_uri_v2 = "/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v2; - String get_enc_response_part1 = "\"pathId\":\"/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v1 + "\""; - String get_enc_response_part1_v2 = "\"pathId\":\"/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v2 + "\""; - - // JSON encode " as \" - String get_enc_response_part2 = "\"id\":\"id:namespace:document-type::" + get_enc_id.replace("\"", "\\\"") + "\""; - - - @Test - public void testbasicEncodingV1() throws Exception { - Request request = new Request("http://localhost:" + getFirstListenPort() + get_enc_test_uri_v1); - HttpGet get = new HttpGet(request.getUri()); - Response response = doRest(get); - assertThat(response.code, is(404)); // Mock returns Not Found - assertThat(response.body, containsString(get_enc_response_part1)); - assertThat(response.body, containsString(get_enc_response_part2)); - } - - @Test - public void testbasicEncodingV2() throws Exception { - Request request = new Request("http://localhost:" + getFirstListenPort() + get_enc_test_uri_v2); - HttpGet get = new HttpGet(request.getUri()); - Response response = doRest(get); - assertThat(response.code, is(404)); // Mock returns Not Found - assertThat(response.body, containsString(get_enc_response_part1_v2)); - assertThat(response.body, containsString(get_enc_response_part2)); - } - - @Test - public void get_fieldset_parameter_is_propagated() { - Request request = new Request(String.format("http://localhost:%s/document/v1/namespace/document-type/docid/bar?fieldSet=foo,baz", getFirstListenPort())); - HttpGet get = new HttpGet(request.getUri()); - assertHttp200ResponseContains(doRest(get), "\"fieldset\":\"foo,baz\""); - } - - @Test - public void get_cluster_parameter_is_propagated() { - Request request = new Request(String.format("http://localhost:%s/document/v1/namespace/document-type/docid/bar?cluster=my_cool_cluster", getFirstListenPort())); - HttpGet get = new HttpGet(request.getUri()); - assertHttp200ResponseContains(doRest(get), "\"cluster\":\"my_cool_cluster\""); - } - - String visit_test_uri = "/document/v1/namespace/document-type/docid/?continuation=abc"; - String visit_response_part1 = "\"documents\":[List of json docs, cont token abc, doc selection: '']"; - String visit_response_part2 = "\"continuation\":\"token\""; - String visit_response_part3 = "\"pathId\":\"/document/v1/namespace/document-type/docid/\""; - - @Test - public void testbasicVisit() throws Exception { - Request request = new Request("http://localhost:" + getFirstListenPort() + visit_test_uri); - HttpGet get = new HttpGet(request.getUri()); - Response response = doRest(get); - assertThat(response.code, is(200)); - assertThat(response.body, containsString(visit_response_part1)); - assertThat(response.body, containsString(visit_response_part2)); - assertThat(response.body, containsString(visit_response_part3)); - } - - private static String encoded(String original) { - try { - return URLEncoder.encode(original, StandardCharsets.UTF_8.name()); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - } - - private static String defaultPathPrefix() { - return "namespace/document-type/"; - } - - private Response performV1RestCall(String pathPrefix, String pathSuffix, Function<Request, HttpRequestBase> methodOp) { - try { - Request request = new Request(String.format("http://localhost:%s/document/v1/%s%s", - getFirstListenPort(), pathPrefix, pathSuffix)); - HttpRequestBase restOp = methodOp.apply(request); - return doRest(restOp); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private Response performV1GetRestCall(String pathSuffix) { - return performV1RestCall(defaultPathPrefix(), pathSuffix, (request) -> new HttpGet(request.getUri())); - } - - private void doTestRootPathNotAccepted(Function<Request, HttpRequestBase> methodOpFactory) { - Response response = performV1RestCall("", "", methodOpFactory); - assertHttp400ResponseContains(response, "Root /document/v1/ requests only supported for HTTP GET"); - } - - @Test - public void root_api_path_not_accepted_for_http_put() { - doTestRootPathNotAccepted((request) -> new HttpPut(request.getUri())); - } - - @Test - public void root_api_path_not_accepted_for_http_post() { - doTestRootPathNotAccepted((request) -> new HttpPost(request.getUri())); - } - - @Test - public void root_api_path_not_accepted_for_http_delete() { - doTestRootPathNotAccepted((request) -> new HttpDelete(request.getUri())); - } - - private void assertResultingDocumentSelection(String suffix, String expected) { - Response response = performV1GetRestCall(suffix); - assertHttp200ResponseContains(response, String.format("doc selection: '%s'", expected)); - } - - @Test - public void testUseExpressionOnVisit() throws Exception { - assertResultingDocumentSelection("group/abc?continuation=xyz", "id.group=='abc'"); - } - - private void assertGroupDocumentSelection(String group, String expected) { - assertResultingDocumentSelection("group/" + encoded(group), expected); - } - - @Test - public void group_strings_are_escaped() { - assertGroupDocumentSelection("'", "id.group=='\\''"); - assertGroupDocumentSelection("hello 'world'", "id.group=='hello \\'world\\''"); - assertGroupDocumentSelection("' goodbye moon", "id.group=='\\' goodbye moon'"); - } - - private void assertNumericIdFailsParsing(String id) { - Response response = performV1GetRestCall(String.format("number/%s", encoded(id))); - assertHttp400ResponseContains(response, "Failed to parse numeric part of selection URI"); - } - - @Test - public void invalid_numeric_id_returns_error() { - assertNumericIdFailsParsing("123a"); - assertNumericIdFailsParsing("a123"); - assertNumericIdFailsParsing("0x1234"); - assertNumericIdFailsParsing("\u0000"); - } - - @Test - public void non_text_group_string_character_returns_error() { - Response response = performV1GetRestCall(String.format("group/%s", encoded("\u001f"))); - assertHttp400ResponseContains(response, "Failed to parse group part of selection URI; contains invalid text code point U001F"); - } - - @Test - public void can_specify_numeric_id_without_explicit_selection() { - assertResultingDocumentSelection("number/1234", "id.user==1234"); - } - - @Test - public void can_specify_group_id_without_explicit_selection() { - assertResultingDocumentSelection("group/foo", "id.group=='foo'"); - } - - @Test - public void can_specify_both_numeric_id_and_explicit_selection() { - assertResultingDocumentSelection(String.format("number/1234?selection=%s", encoded("1 != 2")), - "id.user==1234 and (1 != 2)"); - } - - @Test - public void can_specify_both_group_id_and_explicit_selection() { - assertResultingDocumentSelection(String.format("group/bar?selection=%s", encoded("3 != 4")), - "id.group=='bar' and (3 != 4)"); - } - - private void assertDocumentSelectionFailsParsing(String expression) { - Response response = performV1GetRestCall(String.format("number/1234?selection=%s", encoded(expression))); - assertHttp400ResponseContains(response, "Failed to parse expression given in 'selection' parameter. Must be a complete and valid sub-expression."); - } - - // Make sure that typoing the selection parameter doesn't corrupt the entire selection expression - @Test - public void explicit_selection_sub_expression_is_validated_for_completeness() { - assertDocumentSelectionFailsParsing("1 +"); - assertDocumentSelectionFailsParsing(") or true"); - assertDocumentSelectionFailsParsing("((1 + 2)"); - assertDocumentSelectionFailsParsing("true) or (true"); - } - - @Test - public void wanted_document_count_returned_parameter_is_propagated() { - Request request = new Request(String.format("http://localhost:%s/document/v1/namespace/document-type/docid/?wantedDocumentCount=321", getFirstListenPort())); - HttpGet get = new HttpGet(request.getUri()); - assertHttp200ResponseContains(doRest(get), "min docs returned: 321"); - } - - @Test - public void invalid_wanted_document_count_parameter_returns_error_response() { - Request request = new Request(String.format("http://localhost:%s/document/v1/namespace/document-type/docid/?wantedDocumentCount=aardvark", getFirstListenPort())); - HttpGet get = new HttpGet(request.getUri()); - assertHttp400ResponseContains(doRest(get), "Invalid 'wantedDocumentCount' value. Expected positive integer"); - } - - @Test - public void negative_document_count_parameter_returns_error_response() { - Request request = new Request(String.format("http://localhost:%s/document/v1/namespace/document-type/docid/?wantedDocumentCount=-1", getFirstListenPort())); - HttpGet get = new HttpGet(request.getUri()); - assertHttp400ResponseContains(doRest(get), "Invalid 'wantedDocumentCount' value. Expected positive integer"); - } - - @Test - public void visit_fieldset_parameter_is_propagated() { - Request request = new Request(String.format("http://localhost:%s/document/v1/namespace/document-type/docid/?fieldSet=foo,baz", getFirstListenPort())); - HttpGet get = new HttpGet(request.getUri()); - assertHttp200ResponseContains(doRest(get), "field set: 'foo,baz'"); - } - - @Test - public void visit_concurrency_parameter_is_propagated() { - Request request = new Request(String.format("http://localhost:%s/document/v1/namespace/document-type/docid/?concurrency=42", getFirstListenPort())); - HttpGet get = new HttpGet(request.getUri()); - assertHttp200ResponseContains(doRest(get), "concurrency: 42"); - } - - @Test - public void root_api_visit_cluster_parameter_is_propagated() { - Request request = new Request(String.format("http://localhost:%s/document/v1/?cluster=vaffel", getFirstListenPort())); - HttpGet get = new HttpGet(request.getUri()); - assertHttp200ResponseContains(doRest(get), "cluster: 'vaffel'"); - } - - @Test - public void root_api_visit_selection_parameter_is_propagated() { - Request request = new Request(String.format("http://localhost:%s/document/v1/?cluster=foo&selection=yoshi", getFirstListenPort())); - HttpGet get = new HttpGet(request.getUri()); - assertHttp200ResponseContains(doRest(get), "doc selection: 'yoshi'"); - } - - @Test - public void root_api_visit_bucket_space_parameter_is_propagated() { - Request request = new Request(String.format("http://localhost:%s/document/v1/?cluster=foo&bucketSpace=global", getFirstListenPort())); - HttpGet get = new HttpGet(request.getUri()); - assertHttp200ResponseContains(doRest(get), "bucket space: 'global'"); - } - - @Test - public void invalid_visit_concurrency_parameter_returns_error_response() { - Request request = new Request(String.format("http://localhost:%s/document/v1/namespace/document-type/docid/?concurrency=badgers", getFirstListenPort())); - HttpGet get = new HttpGet(request.getUri()); - assertHttp400ResponseContains(doRest(get), "Invalid 'concurrency' value. Expected positive integer"); - } - - private void assertHttpResponseContains(Response response, int expectedStatusCode, String expectedSubstring) { - assertThat(response.code, is(expectedStatusCode)); - assertThat(response.body, containsString(expectedSubstring)); - } - - private void assertHttp200ResponseContains(Response response, String expectedSubstring) { - assertHttpResponseContains(response, 200, expectedSubstring); - } - - private void assertHttp400ResponseContains(Response response, String expectedSubstring) { - assertHttpResponseContains(response, 400, expectedSubstring); - } - - private Response doRest(HttpRequestBase request) { - HttpClient client = HttpClientBuilder.create().build(); - try { - HttpResponse response = client.execute(request); - assertThat(response.getEntity().getContentType().getValue().toString(), startsWith("application/json;")); - HttpEntity entity = response.getEntity(); - return new Response(response.getStatusLine().getStatusCode(), EntityUtils.toString(entity)); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private String getFirstListenPort() { - JettyHttpServer serverProvider = - (JettyHttpServer) Container.get().getServerProviderRegistry().allComponents().get(0); - return Integer.toString(serverProvider.getListenPort()); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java deleted file mode 100644 index db782877a6f..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.restapi.resource; - -import com.yahoo.container.logging.AccessLog; -import com.yahoo.document.DataType; -import com.yahoo.document.DocumentType; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.restapi.OperationHandler; - -import java.util.concurrent.Executor; - -/** - * For setting up RestApi with a simple document type manager. - * - * @author dybis - */ -public class RestApiWithTestDocumentHandler extends RestApi{ - - private DocumentTypeManager docTypeManager = new DocumentTypeManager(); - - public RestApiWithTestDocumentHandler( - Executor executor, - AccessLog accessLog, - OperationHandler operationHandler) { - super(executor, accessLog, operationHandler, 20); - - DocumentType documentType = new DocumentType("testdocument"); - - documentType.addField("title", DataType.STRING); - documentType.addField("body", DataType.STRING); - docTypeManager.registerDocumentType(documentType); - - setDocTypeManagerForTests(docTypeManager); - } - -} diff --git a/vespaclient-container-plugin/src/test/rest-api-application/services.xml b/vespaclient-container-plugin/src/test/rest-api-application/services.xml deleted file mode 100644 index ae1b87635a9..00000000000 --- a/vespaclient-container-plugin/src/test/rest-api-application/services.xml +++ /dev/null @@ -1,19 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<container version="1.0" jetty="true"> - - <accesslog type="disabled"/> - - <handler id="com.yahoo.document.restapi.resource.RestApiWithTestDocumentHandler" bundle="integration-test"> - <binding>http://*/document/v1/*</binding> - </handler> - - <component id="injected" class="com.yahoo.document.restapi.resource.MockedOperationHandler" bundle="integration-test"> - </component> - - - <http> - <!-- This indicates that we want JDisc to allocate a port for us --> - <server id="mainServer" port="0" /> - </http> -</container> |