diff options
Diffstat (limited to 'vespaclient-container-plugin')
36 files changed, 35 insertions, 6315 deletions
diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml index 150a3241153..3567e6ed1e6 100644 --- a/vespaclient-container-plugin/pom.xml +++ b/vespaclient-container-plugin/pom.xml @@ -1,18 +1,18 @@ <?xml version="1.0"?> <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <!-- Implementation of document-api in the container. --> -<!-- TODO: Rename to container-documentapi on Vespa 7 --> +<!-- TODO: Rename to container-documentapi --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespaclient-container-plugin</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java index f32b3594f28..cfa77455f41 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java @@ -31,18 +31,6 @@ public interface OperationHandler { public final Optional<Integer> concurrency; public final Optional<String> bucketSpace; - /** @deprecated Use a VisitOptions.Builder instead */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public VisitOptions(Optional<String> cluster, Optional<String> continuation, Optional<Integer> wantedDocumentCount) { - this.cluster = cluster; - this.continuation = continuation; - this.wantedDocumentCount = wantedDocumentCount; - this.fieldSet = Optional.empty(); - this.concurrency = Optional.empty(); - this.bucketSpace = Optional.empty(); - } - private VisitOptions(Builder builder) { this.cluster = Optional.ofNullable(builder.cluster); this.continuation = Optional.ofNullable(builder.continuation); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java deleted file mode 100755 index 365f605688c..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.metrics.MetricManager; -import com.yahoo.metrics.MetricSnapshot; -import com.yahoo.text.Utf8String; -import com.yahoo.text.XMLWriter; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; - -/** - * @deprecated Legacy API. Will be removed in Vespa 7 - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class StatusResponse extends HttpResponse { - - MetricManager manager; - int verbosity; - int snapshotTime; - - StatusResponse(MetricManager manager, int verbosity, int snapshotTime) { - super(com.yahoo.jdisc.http.HttpResponse.Status.OK); - this.manager = manager; - this.snapshotTime = snapshotTime; - this.verbosity = verbosity; - } - - @Override - public void render(OutputStream stream) throws IOException { - XMLWriter writer = new XMLWriter(new OutputStreamWriter(stream)); - writer.openTag("status"); - if (verbosity >= 2) { - writer.attribute(new Utf8String("description"), "Metrics since start"); - } - - if (snapshotTime == 0) { - MetricSnapshot snapshot = (new MetricSnapshot( - "Total metrics from start until current time", 0, - manager.getActiveMetrics().getMetrics(), false)); - manager.getTotalMetricSnapshot().addToSnapshot(snapshot, (int)(System.currentTimeMillis() / 1000), false); - snapshot.printXml(manager, "", verbosity, writer); - } else { - try { - manager.getMetricSnapshotSet(snapshotTime).getSnapshot().printXml(manager, "", verbosity, writer); - } catch (Exception e) { - writer.openTag("error"); - writer.attribute(new Utf8String("details"), "No metric snapshot with period " + snapshotTime + - " was found. Legal snapshot periods are: " + manager.getSnapshotPeriods()); - writer.closeTag(); - } - } - writer.closeTag(); - writer.flush(); - } - - @Override - public java.lang.String getContentType() { - return "application/xml"; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java deleted file mode 100755 index 8fe721899f9..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import java.util.concurrent.Executor; -import javax.inject.Inject; - -import com.yahoo.jdisc.Metric; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; - -/** - * @deprecated Legacy API. Will be removed in Vespa 7 - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class VespaFeedHandlerCompatibility extends ThreadedHttpRequestHandler { - - private final VespaFeedHandlerGet getHandler; - private final VespaFeedHandler feedHandler; - - @Inject - public VespaFeedHandlerCompatibility(Executor executor, Metric metric, VespaFeedHandlerGet getHandler, - VespaFeedHandler feedHandler) { - super(executor, metric); - this.getHandler = getHandler; - this.feedHandler = feedHandler; - } - - @Override - public HttpResponse handle(HttpRequest request) { - boolean hasType = request.hasProperty("type"); - // If we have an ID and no document type, redirect to Get - if (request.hasProperty("id") && !hasType) { - return getHandler.handle(request); - } else { - return feedHandler.handle(request); - } - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java deleted file mode 100755 index ab8728e017b..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import java.util.Collections; -import java.util.concurrent.Executor; -import javax.inject.Inject; - -import com.yahoo.jdisc.Metric; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.search.handler.SearchHandler; - -/** - * @deprecated Legacy API. Will be removed in Vespa 7 - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class VespaFeedHandlerGet extends ThreadedHttpRequestHandler { - - private final SearchHandler searchHandler; - - @Inject - public VespaFeedHandlerGet(SearchHandler searchHandler, Executor executor, Metric metric) { - super(executor, metric, true); - this.searchHandler = searchHandler; - } - - @Override - public HttpResponse handle(HttpRequest request) { - return searchHandler.handle(new HttpRequest(request.getJDiscRequest(), request.getData(), Collections.singletonMap("searchChain", "vespaget"))); - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java deleted file mode 100755 index 9baad1a605d..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import com.google.inject.Inject; -import com.yahoo.clientmetrics.RouteMetricSet; -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.protect.Error; -import com.yahoo.document.DocumentId; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.feedapi.SingleSender; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.vespaclient.config.FeederConfig; - -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.util.concurrent.Executor; - -/** - * @deprecated Legacy API. Will be removed in Vespa 7 - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class VespaFeedHandlerRemove extends VespaFeedHandlerBase { - - @Inject - public VespaFeedHandlerRemove(FeederConfig feederConfig, - LoadTypeConfig loadTypeConfig, - DocumentmanagerConfig documentmanagerConfig, - SlobroksConfig slobroksConfig, - ClusterListConfig clusterListConfig, - Executor executor, - Metric metric) throws Exception { - super(feederConfig, loadTypeConfig, documentmanagerConfig, slobroksConfig, clusterListConfig, executor, metric); - } - - VespaFeedHandlerRemove(FeedContext context, Executor executor) throws Exception { - super(context, executor); - } - - @Override - public HttpResponse handle(HttpRequest request) { - if (request.getProperty("status") != null) { - return new MetricResponse(context.getMetrics().getMetricSet()); - } - - MessagePropertyProcessor.PropertySetter properties = getPropertyProcessor().buildPropertySetter(request); - String route = properties.getRoute().toString(); - FeedResponse response = new FeedResponse(new RouteMetricSet(route, null)); - SingleSender sender = new SingleSender(response, getSharedSender(route)); - sender.addMessageProcessor(properties); - - response.setAbortOnFeedError(properties.getAbortOnFeedError()); - - if (request.hasProperty("id")) { - sender.remove(new DocumentId(request.getProperty("id"))); - } else if (request.hasProperty("id[0]")) { - int index = 0; - while (request.hasProperty("id[" + index + "]")) { - sender.remove(new DocumentId(request.getProperty("id[" + index + "]"))); - ++index; - } - } - - if (request.getData() != null) { - try { - String line; - BufferedReader reader = new BufferedReader( - new InputStreamReader(getRequestInputStream(request), "UTF-8")); - while ((line = reader.readLine()) != null) { - sender.remove(new DocumentId(line)); - } - } catch (Exception e) { - response.addError(e.getClass() + ": " + e.getCause()); - } - } - - sender.done(); - long millis = getTimeoutMillis(request); - boolean completed = sender.waitForPending(millis); - if ( ! completed) - response.addError(Error.TIMEOUT, "Timed out after "+millis+" ms waiting for responses"); - return response; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java deleted file mode 100644 index 0ca77decf22..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import com.google.inject.Inject; -import com.yahoo.clientmetrics.RouteMetricSet; -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.protect.Error; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.documentapi.messagebus.protocol.RemoveLocationMessage; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.feedapi.SingleSender; -import com.yahoo.jdisc.Metric; -import com.yahoo.messagebus.routing.Route; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.vespaclient.config.FeederConfig; - -import java.util.concurrent.Executor; - -/** - * @deprecated Legacy API. Will be removed in Vespa 7 - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class VespaFeedHandlerRemoveLocation extends VespaFeedHandlerBase { - - @Inject - public VespaFeedHandlerRemoveLocation(FeederConfig feederConfig, - LoadTypeConfig loadTypeConfig, - DocumentmanagerConfig documentmanagerConfig, - SlobroksConfig slobroksConfig, - ClusterListConfig clusterListConfig, - Executor executor, Metric metric) throws Exception { - super(feederConfig, loadTypeConfig, documentmanagerConfig, slobroksConfig, clusterListConfig, executor, metric); - } - - VespaFeedHandlerRemoveLocation(FeedContext context, Executor executor) throws Exception { - super(context, executor); - } - - @Override - public HttpResponse handle(HttpRequest request) { - MessagePropertyProcessor.PropertySetter properties = getPropertyProcessor().buildPropertySetter(request); - FeedResponse response; - - if (request.getProperty("route") == null) { - if (context.getClusterList().getStorageClusters().size() == 0) { - return new FeedResponse(null).addError("No storage clusters configured and no alternate route specified."); - } else if (context.getClusterList().getStorageClusters().size() > 1) { - return new FeedResponse(null).addError("More than one storage cluster configured and no route specified."); - } else { - properties.setRoute(Route.parse(context.getClusterList().getStorageClusters().get(0).getName())); - } - } - - response = new FeedResponse(new RouteMetricSet(properties.getRoute().toString(), null)); - - SingleSender sender = new SingleSender(response, getSharedSender(properties.getRoute().toString())); - sender.addMessageProcessor(properties); - - String user = request.getProperty("user"); - String group = request.getProperty("group"); - String selection = request.getProperty("selection"); - - boolean oneFound = (user != null) ^ (group != null) ^ (selection != null); - - if (!oneFound) { - response.addError("Exactly one of \"user\", \"group\" or \"selection\" must be specified for removelocation"); - return response; - } - - if (user != null) { - selection = "id.user=" + user; - } - if (group != null) { - selection = "id.group=\"" + group + "\""; - } - - sender.send(new RemoveLocationMessage(selection)); - sender.done(); - long millis = getTimeoutMillis(request); - boolean completed = sender.waitForPending(millis); - if ( ! completed) - response.addError(Error.TIMEOUT, "Timed out after "+millis+" ms waiting for responses"); - return response; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java deleted file mode 100755 index ac16159a9bb..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import java.util.concurrent.Executor; - -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.metrics.MetricManager; -import com.yahoo.metrics.MetricSet; -import com.yahoo.vespaclient.config.FeederConfig; - -/** - * @deprecated Legacy API. Will be removed in Vespa 7 - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class VespaFeedHandlerStatus extends ThreadedHttpRequestHandler { - - private MetricManager manager; - - public VespaFeedHandlerStatus(FeederConfig feederConfig, - LoadTypeConfig loadTypeConfig, - DocumentmanagerConfig documentmanagerConfig, - SlobroksConfig slobroksConfig, - ClusterListConfig clusterListConfig, - Executor executor, - Metric metric) { - this(FeedContext.getInstance(feederConfig, loadTypeConfig, - documentmanagerConfig, slobroksConfig, - clusterListConfig, metric), - true, true, executor); - } - - VespaFeedHandlerStatus(FeedContext context, boolean doLog, boolean makeSnapshots, Executor executor) { - super(executor); - manager = new MetricManager(); - final MetricSet metricSet = context.getMetrics().getMetricSet(); - metricSet.unregister(); - manager.registerMetric(metricSet); - if (doLog) { - manager.addMetricToConsumer("log", "routes.total.putdocument.count"); - manager.addMetricToConsumer("log", "routes.total.removedocument.count"); - manager.addMetricToConsumer("log", "routes.total.updatedocument.count"); - manager.addMetricToConsumer("log", "routes.total.getdocument.count"); - - manager.addMetricToConsumer("log", "routes.total.putdocument.errors.total"); - manager.addMetricToConsumer("log", "routes.total.removedocument.errors.total"); - manager.addMetricToConsumer("log", "routes.total.updatedocument.errors.total"); - manager.addMetricToConsumer("log", "routes.total.getdocument.errors.total"); - - manager.addMetricToConsumer("log", "routes.total.putdocument.latency"); - manager.addMetricToConsumer("log", "routes.total.removedocument.latency"); - manager.addMetricToConsumer("log", "routes.total.updatedocument.latency"); - manager.addMetricToConsumer("log", "routes.total.getdocument.latency"); - } - - if (doLog || makeSnapshots) { - new Thread(manager).start(); - } - } - - @Override - public HttpResponse handle(HttpRequest request) { - try { - return new StatusResponse(manager, asInt(request.getProperty("verbosity"), 0), asInt(request.getProperty("snapshotperiod"), 0)); - } catch (Exception e) { - e.printStackTrace(); - return null; - } - } - - private int asInt(String value, int defaultValue) { - if (value == null) return defaultValue; - return Integer.parseInt(value); - } - - @Override - public void destroy() { - manager.stop(); - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java deleted file mode 100644 index b73ea455c32..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import java.util.Collections; -import java.util.concurrent.Executor; -import javax.inject.Inject; - -import com.yahoo.jdisc.Metric; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.search.handler.SearchHandler; - -/** - * @author thomasg - * - * @deprecated Legacy API. Will be removed in Vespa 7 - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class VespaFeedHandlerVisit extends ThreadedHttpRequestHandler { - - private final SearchHandler searchHandler; - - @Inject - public VespaFeedHandlerVisit(SearchHandler searchHandler, Executor executor, Metric metric) { - super(executor, metric, true); - this.searchHandler = searchHandler; - } - - @Override - public HttpResponse handle(HttpRequest request) { - return searchHandler.handle(new HttpRequest(request.getJDiscRequest(), request.getData(), Collections.singletonMap("searchChain", "vespavisit"))); - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/package-info.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/package-info.java deleted file mode 100644 index 857e7127b6f..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/package-info.java +++ /dev/null @@ -1,3 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// TODO: This implements the old, deprecated document http API. Remove this package on Vespa 7. -package com.yahoo.feedhandler; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java deleted file mode 100755 index c66af4f02ba..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.documentapi.ProgressToken; -import com.yahoo.search.result.Hit; -import java.io.IOException; -import java.util.Base64; - -/** - * @deprecated - */ -@Deprecated // OK -// TODO: Remove on Vespa 7 -public class ContinuationHit extends Hit { - - private final String value; - - public ContinuationHit(ProgressToken token) { - super("continuation"); - value = token.serializeToString(); - } - - public static ProgressToken getToken(String continuation) { - return ProgressToken.fromSerializedString(continuation); - } - - public String getValue() { - return value; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java deleted file mode 100755 index cf35c789f65..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.DataType; -import com.yahoo.document.Document; -import com.yahoo.document.Field; -import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.document.datatypes.Raw; -import com.yahoo.io.ByteWriter; -import com.yahoo.text.XML; - -import java.io.IOException; -import java.io.Writer; - -/** - * Template used to render a single field for a single Document. Fields - * that are either of type CONTENT or RAW are written directly, while - * all other fields are wrapped in Vespa XML and escaped. - * - * @deprecated use a renderer instead - */ -@Deprecated // OK -// TODO: Remove on Vespa 7 -@SuppressWarnings("deprecation") -public class DocumentFieldTemplate extends com.yahoo.prelude.templates.UserTemplate<Writer> { - - Field field; - String contentType; - String encoding; - boolean wrapXml; - - public DocumentFieldTemplate(Field field, String contentType, String encoding, boolean wrapXml) { - super("documentfield", contentType, encoding); - this.field = field; - this.contentType = contentType; - this.encoding = encoding; - this.wrapXml = wrapXml; - } - - @Override - public void error(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - // Error shouldn't be handled by this template, but rather - // delegated to the searcher - } - - @Override - public Writer wrapWriter(Writer writer) { - /* TODO: uncomment - if (!(writer instanceof ByteWriter)) { - throw new IllegalArgumentException("ByteWriter required, but got " + writer.getClass().getName()); - } - */ - - return writer; - } - - @Override - public void header(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - if (wrapXml) { - // XML wrapping should only be used for default field rendering - writer.write("<?xml version=\"1.0\" encoding=\"" + encoding + "\"?>\n"); - writer.write("<result>"); - } - } - - @Override - public void footer(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - if (wrapXml) { - writer.write("</result>\n"); - } - } - - @Override - public void hit(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - DocumentHit hit = (DocumentHit)context.get("hit"); - Document doc = hit.getDocument(); - // Assume field existence has been checked before we ever get here. - // Also assume that relevant encoding/content type is set - // appropriately according to the request and the field's content - // type, as this is immutable in the template set. - FieldValue value = doc.getFieldValue(field); - if (field.getDataType() == DataType.RAW) { - ByteWriter bw = (ByteWriter)writer; - bw.append(((Raw) value).getByteBuffer().array()); - } else { - writer.write(XML.xmlEscape(value.toString(), false)); - } - } - - @Override - public void hitFooter(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - } - - @Override - public void noHits(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java deleted file mode 100755 index 514cf055e84..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.Document; -import com.yahoo.document.Field; -import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.search.result.Hit; - -import java.util.Iterator; -import java.util.Map; - -@Deprecated // OK -// TODO: Remove on Vespa 7 -public class DocumentHit extends Hit { - - private Document document; - private int index; - - public DocumentHit(Document document, int index) { - super(document.getId().toString()); - this.document = document; - this.index = index; - } - - public void populateHitFields() { - // Create hit fields for all document fields - Iterator<Map.Entry<Field, FieldValue>> fieldIter = document.iterator(); - while (fieldIter.hasNext()) { - Map.Entry<Field, FieldValue> field = fieldIter.next(); - setField(field.getKey().getName(), field.getValue()); - } - - // Assign an explicit document id field - setField("documentid", document.getId().toString()); - } - - public Document getDocument() { - return document; - } - - public int getIndex() { - return index; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java deleted file mode 100644 index 73363e68e2b..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.DocumentId; -import com.yahoo.search.result.Hit; - -@Deprecated // OK -// TODO: Remove on Vespa 7 -public class DocumentRemoveHit extends Hit { - - private final DocumentId idOfRemovedDoc; - - public DocumentRemoveHit(DocumentId idOfRemovedDoc) { - super(idOfRemovedDoc.toString()); - this.idOfRemovedDoc = idOfRemovedDoc; - setField("documentid", idOfRemovedDoc.toString()); - } - - public DocumentId getIdOfRemovedDoc() { - return idOfRemovedDoc; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java deleted file mode 100755 index 2366e7059bd..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.log.LogLevel; -import com.yahoo.search.Result; -import com.yahoo.search.result.ErrorHit; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.result.Hit; -import com.yahoo.text.XML; - -import java.io.IOException; -import java.io.Writer; -import java.util.logging.Logger; - -/** - * @deprecated use a renderer instead - */ -@Deprecated // OK -// TODO: Remove on Vespa 7 -@SuppressWarnings("deprecation") -public class DocumentXMLTemplate extends com.yahoo.prelude.templates.UserTemplate<Writer> { - - private static final Logger log = Logger.getLogger(DocumentXMLTemplate.class.getName()); - - public DocumentXMLTemplate() { - super("vespa_xml"); - } - - public DocumentXMLTemplate(String mimeType, String encoding) { - super("vespa_xml", mimeType, encoding); - } - - private void writeErrorMessage(Writer writer, String type, int code, - String message, String detailedMessage) throws IOException { - writer.write("<error type=\"" + type + "\" code=\"" + code + "\" message=\""); - writer.write(XML.xmlEscape(message, true)); - if (detailedMessage != null) { - writer.write(": "); - writer.write(XML.xmlEscape(detailedMessage, true)); - } - writer.write("\"/>\n"); - } - - private void writeGenericErrorMessage(Writer writer, ErrorMessage message) throws IOException { - // A bit dirty, but we don't have to support many different types - if (message instanceof MessageBusErrorMessage) { - writeErrorMessage(writer, "messagebus", - ((MessageBusErrorMessage)message).getMessageBusCode(), - message.getMessage(), message.getDetailedMessage()); - } else { - writeErrorMessage(writer, "searcher", message.getCode(), - message.getMessage(), message.getDetailedMessage()); - } - } - - @Override - public void error(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - writer.write("<errors>\n"); - // If the error contains no error hits, use a single error with the main - // code and description. Otherwise, use the error hits explicitly - ErrorHit errorHit = ((Result)context.get("result")).hits().getErrorHit(); - if (errorHit == null || errorHit.errors().isEmpty()) { - ErrorMessage message = ((Result)context.get("result")).hits().getError(); - writeGenericErrorMessage(writer, message); - } else { - for (ErrorMessage message : errorHit.errors()) { - writeGenericErrorMessage(writer, message); - } - } - writer.write("</errors>\n"); - } - - @Override - public void header(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); - writer.write("<result>\n"); - HitGroup rootGroup = ((Result) context.get("result")).hits(); - if (rootGroup.getField(VisitSearcher.VISITOR_CONTINUATION_TOKEN_FIELDNAME) != null) { - writer.write("<continuation>" + rootGroup.getField(VisitSearcher.VISITOR_CONTINUATION_TOKEN_FIELDNAME) + "</continuation>"); - } - } - - @Override - public void footer(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - writer.write("</result>\n"); - } - - @Override - public void hit(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - Hit hit = (Hit)context.get("hit"); - if (hit instanceof DocumentHit) { - DocumentHit docHit = (DocumentHit) hit; - if (docHit.getDocument() != null) { - writer.write(docHit.getDocument().toXML(" ")); - } - } else if (hit instanceof DocumentRemoveHit) { - writeDocumentRemoveHit(writer, (DocumentRemoveHit) hit); - } else { - log.log(LogLevel.WARNING, "Cannot render document XML; expected hit of type " + - "com.yahoo.storage.searcher.Document[Remove]Hit, got " + hit.getClass().getName() + - ". Is there another backend searcher present?"); - } - } - - private void writeDocumentRemoveHit(Writer writer, DocumentRemoveHit remove) throws IOException { - writer.write("<remove documentid=\""); - writer.write(XML.xmlEscape(remove.getIdOfRemovedDoc().toString())); - writer.write("\"/>\n"); - } - - @Override - public void hitFooter(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - } - - @Override - public void noHits(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java deleted file mode 100755 index 869a2f9317a..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java +++ /dev/null @@ -1,508 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.google.inject.Inject; -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.jdisc.Metric; -import com.yahoo.processing.request.CompoundName; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.document.DataType; -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.document.Field; -import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; -import com.yahoo.feedapi.*; -import com.yahoo.log.LogLevel; -import com.yahoo.messagebus.Reply; -import com.yahoo.search.Query; -import com.yahoo.search.query.Properties; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.result.DefaultErrorHit; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.vespaclient.config.FeederConfig; - -import java.io.*; -import java.util.*; -import java.util.logging.Logger; -import java.util.zip.GZIPInputStream; - -/** - * Searcher component to make GET requests to a content cluster. - * <p> - * Document ID must be given either as 1 "id=docid" query parameter - * for single-document GETs and 1-n "id[0]=docid_1&id[1]=...&id[n-1]=docid_n" - * parameters for multi-document GETs. - * - * <p> - * Standard gateway query parameters are implicitly supported: - * priority, timeout, route - * - * <p> - * The searcher also accepts the following (optional) query parameters: - * headersonly=true|false (default: false) - * For specifying whether or not to return only header fields. - * - * <p> - * field=string (default: no parameter specified) - * For getting a single document field. - * - * <p> - * contenttype=string (default: no content type specified) - * For specifiying the returned HTTP header content type for a returned - * document field's content. field must also be specified. - * @deprecated do not use - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class GetSearcher extends Searcher { - - private static final Logger log = Logger.getLogger(GetSearcher.class.getName()); - - private static final CompoundName ID = new CompoundName("id"); - private static final CompoundName HEADERS_ONLY = new CompoundName("headersonly"); - private static final CompoundName POPULATE_HIT_FIELDS = new CompoundName("populatehitfields"); - private static final CompoundName FIELDSET = new CompoundName("fieldset"); - private static final CompoundName FIELD = new CompoundName("field"); - private static final CompoundName CONTENT_TYPE = new CompoundName("contenttype"); - private static final CompoundName TIEMOUT = new CompoundName("timeout"); - - FeedContext context; - - private final long defaultTimeoutMillis; - - private class GetResponse implements SharedSender.ResultCallback { - - /** - * We have to maintain the same ordering of results as that - * given in the request. Do this by remembering the index of - * each requested document ID. - */ - private Map<String, Integer> ordering; - private List<DocumentHit> documentHits = new ArrayList<>(); - private List<DefaultErrorHit> errorHits = new ArrayList<>(); - private List<Reply> replies = new ArrayList<>(); - private final SharedSender.Pending pendingNumber = new SharedSender.Pending(); - - public GetResponse(List<String> documentIds) { - ordering = new HashMap<>(documentIds.size()); - for (int i = 0; i < documentIds.size(); ++i) { - ordering.put(documentIds.get(i), i); - } - } - - public boolean isAborted() { - return false; - } - - private String stackTraceFromException(Exception e) { - StringWriter sw = new StringWriter(); - PrintWriter ps = new PrintWriter(sw); - e.printStackTrace(ps); - ps.flush(); - return sw.toString(); - } - - public boolean handleReply(Reply reply) { - if ((reply.getTrace().getLevel() > 0) && log.isLoggable(LogLevel.DEBUG)) { - String str = reply.getTrace().toString(); - log.log(LogLevel.DEBUG, str); - } - replies.add(reply); - return true; - } - public SharedSender.Pending getPending() { return pendingNumber; } - - private void processReplies() { - for (Reply reply : replies) { - processReply(reply); - } - } - private void processReply(Reply reply) { - if (!reply.hasErrors()) { - try { - addDocumentHit(reply); - } catch (Exception e) { - String msg = "Got exception of type " + e.getClass().getName() - + " during document deserialization: " + e.getMessage(); - errorHits.add(new DefaultErrorHit("GetSearcher", ErrorMessage.createInternalServerError(msg))); - log.log(LogLevel.DEBUG, "Got exception during document deserialization: " + stackTraceFromException(e)); - } - } else { - errorHits.add(new DefaultErrorHit("GetSearcher", new MessageBusErrorMessage( - reply.getError(0).getCode(), 0, reply.getError(0).getMessage()))); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Received error reply with message " + reply.getError(0).getMessage()); - } - } - } - - private void addDocumentHit(Reply reply) { - Document doc = ((GetDocumentReply)reply).getDocument(); - GetDocumentMessage msg = (GetDocumentMessage)reply.getMessage(); - Integer index = ordering.get(msg.getDocumentId().toString()); - if (index == null) { // Shouldn't happen - throw new IllegalStateException("Received GetDocumentReply for unknown document: " - + doc.getId().toString()); - } - if (doc != null) { - documentHits.add(new DocumentHit(doc, index)); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Received GetDocumentReply for " - + doc.getId().toString()); - } - } else { - // Don't add a hit for documents that can't be found - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Received empty (not found) GetDocumentReply for " - + msg.getDocumentId().toString()); - } - } - } - - private class IndexComparator implements Comparator<DocumentHit> { - public int compare(DocumentHit o1, DocumentHit o2) { - return o1.getIndex() - o2.getIndex(); - } - } - - public void addHitsToResult(Result result, boolean populateHitFields) { - for (DefaultErrorHit hit : errorHits) { - result.hits().add(hit); - } - // Sort document hits according to their request index - Collections.sort(documentHits, new IndexComparator()); - for (DocumentHit hit : documentHits) { - if (populateHitFields) { - hit.populateHitFields(); - } - result.hits().add(hit); - } - result.setTotalHitCount(documentHits.size()); - } - - public List<DocumentHit> getDocumentHits() { - return documentHits; - } - - public List<DefaultErrorHit> getErrorHits() { - return errorHits; - } - } - - @Inject - public GetSearcher(FeederConfig feederConfig, - LoadTypeConfig loadTypeConfig, - DocumentmanagerConfig documentmanagerConfig, - SlobroksConfig slobroksConfig, - ClusterListConfig clusterListConfig, - Metric metric) - throws Exception - { - this(FeedContext.getInstance(feederConfig, loadTypeConfig, - documentmanagerConfig, slobroksConfig, - clusterListConfig, metric), - (long)(feederConfig.timeout() * 1000)); - } - - GetSearcher(FeedContext context) throws Exception { - this.context = context; - this.defaultTimeoutMillis = context.getPropertyProcessor().getDefaultTimeoutMillis(); - } - - GetSearcher(FeedContext context, long defaultTimeoutMillis) throws Exception { - this.context = context; - this.defaultTimeoutMillis = defaultTimeoutMillis; - } - - private static void postValidateDocumentIdParameters(Properties properties, int arrayIdsFound) throws Exception { - for (Map.Entry<String, Object> kv : properties.listProperties().entrySet()) { - if (!kv.getKey().startsWith("id[")) { - continue; - } - if (!kv.getKey().endsWith("]")) { - throw new IllegalArgumentException("Malformed document ID array parameter"); - } - String indexStr = kv.getKey().substring(3, kv.getKey().length() - 1); - int idx = Integer.parseInt(indexStr); - if (idx >= arrayIdsFound) { - throw new IllegalArgumentException("query contains document ID array " + - "that is not zero-based and/or linearly increasing"); - } - } - } - - private List<String> getDocumentIds(Query query) throws Exception { - Properties properties = query.properties(); - List<String> docIds = new ArrayList<>(); - - // First check for regular "id=XX" syntax. If found, return vector with that - // document id only - String singleId = properties.getString(ID); - - int index = 0; - if (singleId != null) { - docIds.add(singleId); - } else { - // Check for id[0]=XX&id[1]=YY...id[n]=ZZ syntax. Indices always start - // at 0 and are always increased by 1. - while (true) { - String docId = properties.getString("id[" + index + "]"); - if (docId == null) { - break; - } - docIds.add(docId); - ++index; - } - postValidateDocumentIdParameters(properties, index); - } - - handleData(query.getHttpRequest(), docIds); - return docIds; - } - - private void handleData(HttpRequest request, List<String> docIds) throws IOException { - if (request.getData() != null) { - InputStream input; - if ("gzip".equals(request.getHeader("Content-Encoding"))) { - input = new GZIPInputStream(request.getData()); - } else { - input = request.getData(); - } - InputStreamReader reader = new InputStreamReader(input, "UTF-8"); - BufferedReader lineReader = new BufferedReader(reader); - String line; - while ((line = lineReader.readLine()) != null) { - docIds.add(line); - } - } - } - - private void handleFieldFiltering(GetResponse response, Result result, - String fieldName, String contentType, - boolean headersOnly) { - - if (response.getDocumentHits().isEmpty()) { - result.hits().addError(ErrorMessage.createNotFound( - "Document not found, could not return field '" + fieldName + "'")); - return; - } - - if (result.hits().getErrorHit() == null) { - Document doc = response.getDocumentHits().get(0).getDocument(); - Field field = doc.getDataType().getField(fieldName); - boolean wrapXml = false; - - if (field == null) { - result.hits().addError(ErrorMessage.createIllegalQuery( - "Field '" + fieldName + "' not found in document type")); - return; - } - FieldValue value = doc.getFieldValue(field); - // If the field exists but hasn't been set in this document, the - // content will be null. We treat this as an error. - if (value == null) { - if (!field.isHeader() && headersOnly) { - // TODO(vekterli): make this work with field sets as well. - result.hits().addError(ErrorMessage.createInvalidQueryParameter( - "Field '" + fieldName + "' is located in document body, but headersonly " - + "prevents it from being retrieved in " + doc.getId().toString())); - } else { - result.hits().addError(ErrorMessage.createNotFound( - "Field '" + fieldName + "' found in document type, but had " - + "no content in " + doc.getId().toString())); - } - return; - } - String encoding = null; - if (field.getDataType() == DataType.RAW) { - if (contentType == null) { - contentType = "application/octet-stream"; - } - encoding = "ISO-8859-1"; - } else { - // By default, return field wrapped in a blanket of vespa XML - contentType = "text/xml"; - wrapXml = true; - } - if (encoding == null) { - // Encoding doesn't matter for binary content, since we're always - // writing directly to the byte buffer and not through a charset - // encoder. Presumably, the client is intelligent enough to not - // attempt to UTF-8 decode binary data. - encoding = "UTF-8"; - } - // Add hit now that we know there aren't any field errors. Otherwise, - // there would be both an error hit and a document hit in the result - response.addHitsToResult(result, false); - // Override Vespa XML template - result.getTemplating().setTemplates(new DocumentFieldTemplate(field, contentType, encoding, wrapXml)); - } - // else: return with error hit, invoking regular Vespa XML error template - } - - private void validateParameters(String fieldName, String contentType, - List<String> documentIds) { - // Content-type only makes sense for single document queries with a field - // set - if (contentType != null) { - if (documentIds.size() > 1) { - throw new IllegalArgumentException( - "contenttype parameter only valid for single document id query"); - } - if (fieldName == null) { - throw new IllegalArgumentException( - "contenttype set without document field being specified"); - } - } - if (fieldName != null && documentIds.size() > 1) { - throw new IllegalArgumentException( - "Field only valid for single document id query"); - } - } - - // For unit testing - protected MessagePropertyProcessor getMessagePropertyProcessor() { - return context.getPropertyProcessor(); - } - - private void doGetDocuments(Query query, Result result, List<String> documentIds) { - GetResponse response = new GetResponse(documentIds); - Properties properties = query.properties(); - - boolean headersOnly = properties.getBoolean(HEADERS_ONLY, false); - boolean populateHitFields = properties.getBoolean(POPULATE_HIT_FIELDS, false); - String fieldSet = properties.getString(FIELDSET); - String fieldName = properties.getString(FIELD); - String contentType = properties.getString(CONTENT_TYPE); - long timeoutMillis = properties.getString(TIEMOUT) != null ? query.getTimeout() : defaultTimeoutMillis; - - if (fieldSet == null) { - fieldSet = headersOnly ? "[header]" : "[all]"; - } - - validateParameters(fieldName, contentType, documentIds); - - MessagePropertyProcessor.PropertySetter propertySetter; - propertySetter = context.getPropertyProcessor().buildPropertySetter(query.getHttpRequest()); - - SingleSender sender = new SingleSender(response, context.getSharedSender(propertySetter.getRoute().toString())); - sender.addMessageProcessor(propertySetter); - - sendDocumentGetMessages(documentIds, fieldSet, sender); - // Twiddle thumbs until we've received a reply for all documents - sender.done(); - boolean completed = sender.waitForPending(timeoutMillis); - if ( ! completed) { - result.hits().addError(ErrorMessage.createTimeout( - "Timed out after waiting "+timeoutMillis+" ms for responses")); - } - response.processReplies(); - if (fieldName != null) { - handleFieldFiltering(response, result, fieldName, contentType, headersOnly); - } else { - response.addHitsToResult(result, populateHitFields); - } - } - - private void sendDocumentGetMessages(List<String> documentIds, String fieldSet, SingleSender sender) { - for (String docIdStr : documentIds) { - DocumentId docId = new DocumentId(docIdStr); - GetDocumentMessage getMsg = new GetDocumentMessage(docId, fieldSet); - - sender.send(getMsg); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Sent GetDocumentMessage for " - + docId.toString()); - } - } - } - - boolean verifyBackendDocumentHitsOnly(Result result) { - if (result.hits().size() != 0) { - log.log(LogLevel.DEBUG, "Result had hits after being sent down"); - for (int i = 0; i < result.hits().size(); ++i) { - if (!(result.hits().get(i) instanceof DocumentHit)) { - log.log(LogLevel.WARNING, "Got hit from backend searcher which was " - + "not a com.yahoo.storage.searcher.DocumentHit instance: " - + result.hits().get(i).getClass().getName()); - return false; - } - } - } - return true; - } - - @Override - public Result search(Query query, Execution execution) { - // Pass through to next searcher - Result result = execution.search(query); - - List<String> documentIds; - try { - documentIds = getDocumentIds(query); - } catch (Exception e) { - setOutputFormat(query, result); - result.hits().addError(ErrorMessage.createIllegalQuery(e.getClass().getName() + ": " + e.getMessage())); - return result; - } - // Early-out for pass-through queries - if (documentIds.isEmpty()) { - return result; - } - // Make sure we don't try to combine non-document hits and document hits - // in the same result set. - if (!verifyBackendDocumentHitsOnly(result)) { - result = new Result(query); // Don't include unknown hits - setOutputFormat(query, result); - result.hits().addError(ErrorMessage.createInternalServerError( - "A backend searcher to com.yahoo.storage.searcher.GetSearcher " + - "returned a hit that was not an instance of com.yahoo.storage.searcher.DocumentHit. " + - "Only DocumentHit instances are supported in the backend hit result set when doing " + - "queries that contain document identifier sets recognised by the Get Searcher.")); - return result; - } - setOutputFormat(query, result); - // Do not propagate exceptions back up, as we want to have all errors - // be reported using the proper template - try { - doGetDocuments(query, result, documentIds); - query.setHits(result.hits().size()); - } catch (IllegalArgumentException e) { - result.hits().addError(ErrorMessage.createIllegalQuery(e.getClass().getName() + ": " + e.getMessage())); - } catch (Exception e) { - result.hits().addError(ErrorMessage.createUnspecifiedError(e.getClass().getName() + ": " + e.getMessage())); - } - - return result; - } - - private static final CompoundName formatShortcut = new CompoundName("format"); - private static final CompoundName format = new CompoundName("presentation.format"); - - /** - * Use custom XML output format unless the default JSON renderer is specified in the request. - */ - @SuppressWarnings("deprecation") - static void setOutputFormat(Query query, Result result) { - if (getRequestProperty(formatShortcut, "", query).equals("JsonRenderer")) return; - if (getRequestProperty(format, "", query).equals("JsonRenderer")) return; - if (getRequestProperty(formatShortcut, "", query).equals("json")) return; - if (getRequestProperty(format, "", query).equals("json")) return; - result.getTemplating().setTemplates(new DocumentXMLTemplate()); - } - - private static String getRequestProperty(CompoundName propertyName, String defaultValue, Query query) { - String propertyValue = query.getHttpRequest().getProperty(propertyName.toString()); - if (propertyValue == null) return defaultValue; - return propertyValue; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java deleted file mode 100755 index ad93645e55e..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.search.result.ErrorMessage; - -/** - * Simple ErrorMessage extension that includes a message bus error code, not - * just the searcher error code (which isn't very useful for a Vespa XML consumer) - */ -@Deprecated // OK -// TODO: Remove on Vespa 7 -public class MessageBusErrorMessage extends ErrorMessage { - - private int mbusCode; - - public MessageBusErrorMessage(int mbusCode, int qrsCode, String message) { - super(qrsCode, message); - this.mbusCode = mbusCode; - } - - public MessageBusErrorMessage(int mbusCode, int qrsCode, String message, String detailedMessage) { - super(qrsCode, message, detailedMessage); - this.mbusCode = mbusCode; - } - - public MessageBusErrorMessage(int mbusCode, int qrsCode, String message, String detailedMessage, Throwable cause) { - super(qrsCode, message, detailedMessage, cause); - this.mbusCode = mbusCode; - } - - public int getMessageBusCode() { - return mbusCode; - } - - public void setMessageBusCode(int code) { - this.mbusCode = code; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java deleted file mode 100644 index e330b76fa4b..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.component.ComponentId; -import com.yahoo.component.ComponentSpecification; -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.documentapi.*; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.jdisc.Metric; -import com.yahoo.messagebus.StaticThrottlePolicy; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.vdslib.VisitorOrdering; -import com.yahoo.vespaclient.ClusterDef; -import com.yahoo.vespaclient.config.FeederConfig; - -/** - * A searcher that allows you to iterate through a storage cluster using visiting. - * - * @deprecated do not use - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class VisitSearcher extends Searcher { - - public static final String VISITOR_CONTINUATION_TOKEN_FIELDNAME = "visitorContinuationToken"; - FeedContext context; - - public VisitSearcher(FeederConfig feederConfig, - LoadTypeConfig loadTypeConfig, - DocumentmanagerConfig documentmanagerConfig, - SlobroksConfig slobroksConfig, - ClusterListConfig clusterListConfig, - Metric metric) - throws Exception - { - this(FeedContext.getInstance(feederConfig, loadTypeConfig, - documentmanagerConfig, slobroksConfig, - clusterListConfig, metric)); - } - - VisitSearcher(FeedContext context) throws Exception { - this.context = context; - } - - class HitDataHandler extends DumpVisitorDataHandler { - private final Result result; - private final boolean populateHits; - private final Object monitor = new Object(); - - HitDataHandler(Result result, boolean populateHits) { - this.result = result; - this.populateHits = populateHits; - } - - @Override - public void onDocument(Document document, long l) { - final DocumentHit hit = new DocumentHit(document, 0); - if (populateHits) { - hit.populateHitFields(); - } - synchronized (monitor) { - result.hits().add(hit); - } - } - - @Override - public void onRemove(DocumentId documentId) { - final DocumentRemoveHit hit = new DocumentRemoveHit(documentId); - synchronized (monitor) { - result.hits().add(hit); - } - } - } - - public VisitorParameters getVisitorParameters(Query query, Result result) throws Exception { - String documentSelection = query.properties().getString("visit.selection"); - if (documentSelection == null) { - documentSelection = ""; - } - - VisitorParameters params = new VisitorParameters(documentSelection); - params.setMaxBucketsPerVisitor(query.properties().getInteger("visit.maxBucketsPerVisitor", 1)); - params.setMaxPending(query.properties().getInteger("visit.maxPendingMessagesPerVisitor", 32)); - params.setMaxFirstPassHits(query.properties().getInteger("visit.approxMaxDocs", 1)); - params.setMaxTotalHits(query.properties().getInteger("visit.approxMaxDocs", 1)); - params.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount( - query.properties().getInteger("visit.maxPendingVisitors", 1))); - params.setToTimestamp(query.properties().getLong("visit.toTimestamp", 0L)); - params.setFromTimestamp(query.properties().getLong("visit.fromTimestamp", 0L)); - - String pri = query.properties().getString("visit.priority"); - if (pri != null) { - params.setPriority(DocumentProtocol.Priority.valueOf(pri)); - } - - if (query.properties().getBoolean("visit.visitInconsistentBuckets")) { - params.visitInconsistentBuckets(true); - } - - String ordering = query.properties().getString("visit.order"); - if (!"ascending".equalsIgnoreCase(ordering)) { - params.setVisitorOrdering(VisitorOrdering.ASCENDING); - } else { - params.setVisitorOrdering(VisitorOrdering.DESCENDING); - } - - String remoteCluster = query.properties().getString("visit.dataHandler"); - if (remoteCluster != null) { - params.setRemoteDataHandler(remoteCluster); - } else { - params.setLocalDataHandler(new HitDataHandler( - result, query.properties().getBoolean("populatehitfields", false))); - } - - String fieldSet = query.properties().getString("visit.fieldSet"); - if (fieldSet != null) { - params.fieldSet(fieldSet); - } - - String continuation = query.properties().getString("visit.continuation"); - if (continuation != null) { - params.setResumeToken(ContinuationHit.getToken(continuation)); - } - - params.setVisitRemoves(query.properties().getBoolean("visit.visitRemoves")); - - MessagePropertyProcessor.PropertySetter propertySetter; - propertySetter = context.getPropertyProcessor().buildPropertySetter(query.getHttpRequest()); - - propertySetter.process(params); - - if (context.getClusterList().getStorageClusters().size() == 0) { - throw new IllegalArgumentException("No content clusters have been defined"); - } - - String route = query.properties().getString("visit.cluster"); - ClusterDef found = null; - if (route != null) { - String names = ""; - for (ClusterDef c : context.getClusterList().getStorageClusters()) { - if (c.getName().equals(route)) { - found = c; - } - if (!names.isEmpty()) { - names += ", "; - } - names += c.getName(); - } - if (found == null) { - throw new IllegalArgumentException("Your vespa cluster contains the storage clusters " + names + ", not " + route + ". Please select a valid vespa cluster."); - } - } else if (context.getClusterList().getStorageClusters().size() == 1) { - found = context.getClusterList().getStorageClusters().get(0); - } else { - throw new IllegalArgumentException("Multiple content clusters are defined, select one using the \"visit.cluster\" option"); - } - - params.setRoute("[Storage:cluster=" + found.getName() + ";clusterconfigid=" + found.getConfigId() + "]"); - return params; - } - - @Override - public Result search(Query query, Execution execution) { - Result result = execution.search(query); - - VisitorParameters parameters; - - try { - parameters = getVisitorParameters(query, result); - } catch (Exception e) { - return new Result(query, ErrorMessage.createBadRequest("Illegal parameters: " + e.toString())); - } - - if (parameters != null) { - VisitorSession session = context.getSessionFactory().createVisitorSession(parameters); - - try { - if (!session.waitUntilDone(query.getTimeout())) { - return new Result(query, ErrorMessage.createTimeout("Visitor timed out")); - } - - ProgressToken token = session.getProgress(); - if (!token.isFinished()) { - final ContinuationHit continuation = new ContinuationHit(token); - result.hits().setField(VISITOR_CONTINUATION_TOKEN_FIELDNAME, continuation.getValue()); - } - } catch (InterruptedException e) { - } finally { - session.destroy(); - } - } - - GetSearcher.setOutputFormat(query, result); - query.setHits(result.hits().size()); - return result; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java deleted file mode 100644 index 581afb0b1c8..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// TODO: This implements the old, deprecated document http API. Remove this package on Vespa 7. -@ExportPackage -package com.yahoo.storage.searcher; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java index 5294545ad50..bd7d195b48b 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java @@ -51,16 +51,8 @@ import java.util.zip.GZIPInputStream; */ public class FeedHandler extends LoggingRequestHandler { - private final ExecutorService workers = Executors.newCachedThreadPool(ThreadFactoryFactory.getThreadFactory("feedhandler")); - private final DocumentTypeManager docTypeManager; - private final Map<String, ClientState> clients; - private final ScheduledThreadPoolExecutor cron; - private final SessionCache sessionCache; protected final ReplyHandler feedReplyHandler; - private final AtomicLong sessionId; - private final Metric metric; - private static final List<Integer> serverSupportedVersions = Collections.unmodifiableList(Arrays.asList(2, 3)); - private final String localHostname; + private static final List<Integer> serverSupportedVersions = Collections.unmodifiableList(Arrays.asList(3)); private final FeedHandlerV3 feedHandlerV3; @Inject @@ -73,46 +65,7 @@ public class FeedHandler extends LoggingRequestHandler { super(parentCtx); DocumentApiMetrics metricsHelper = new DocumentApiMetrics(metricReceiver, "vespa.http.server"); feedHandlerV3 = new FeedHandlerV3(parentCtx, documentManagerConfig, sessionCache, threadpoolConfig, metricsHelper); - docTypeManager = createDocumentManager(documentManagerConfig); - clients = new HashMap<>(); - this.sessionCache = sessionCache; - sessionId = new AtomicLong(ThreadLocalRandom.current().nextLong()); feedReplyHandler = new FeedReplyReader(parentCtx.getMetric(), metricsHelper); - cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feedhandler.cron")); - cron.scheduleWithFixedDelay(new CleanClients(), 16, 11, TimeUnit.MINUTES); - this.metric = parentCtx.getMetric(); - this.localHostname = resolveLocalHostname(); - } - - /** - * Exposed for creating mocks. - */ - protected DocumentTypeManager createDocumentManager(DocumentmanagerConfig documentManagerConfig) { - return new DocumentTypeManager(documentManagerConfig); - } - - private class CleanClients implements Runnable { - - @Override - public void run() { - List<ClientState> clientsToShutdown = new ArrayList<>(); - long now = System.currentTimeMillis(); - - synchronized (clients) { - for (Iterator<Map.Entry<String, ClientState>> i = clients - .entrySet().iterator(); i.hasNext();) { - ClientState client = i.next().getValue(); - - if (now - client.creationTime > 10 * 60 * 1000) { - clientsToShutdown.add(client); - i.remove(); - } - } - } - for (ClientState client : clientsToShutdown) { - client.sourceSession.getReference().close(); - } - } } private Tuple2<HttpResponse, Integer> checkProtocolVersion(HttpRequest request) { @@ -135,8 +88,6 @@ public class FeedHandler extends LoggingRequestHandler { int version; if (washedClientVersions.contains("3")) { version = 3; - } else if (washedClientVersions.contains("2")) { // TODO: Vespa 7: Remove support for Version 2 - version = 2; } else { return new Tuple2<>(new ErrorHttpResponse( Headers.HTTP_NOT_ACCEPTABLE, @@ -175,43 +126,7 @@ public class FeedHandler extends LoggingRequestHandler { if (protocolVersion.first != null) { return protocolVersion.first; } - if (3 == protocolVersion.second) { - return feedHandlerV3.handle(request); - } - final BlockingQueue<OperationStatus> operations = new LinkedBlockingQueue<>(); - Tuple2<String, Boolean> clientId; - clientId = sessionId(request); - - if (clientId.second != null && clientId.second) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Received initial request from client with session ID " + - clientId.first + ", protocol version " + protocolVersion.second); - } - } - - Feeder feeder; - try { - feeder = createFeeder(request, request.getData(), operations, clientId.first, - clientId.second, protocolVersion.second); - // the synchronous FeedResponse blocks draining the InputStream, letting the Feeder read it - workers.submit(feeder); - } catch (UnknownClientException uce) { - String msg = Exceptions.toMessageString(uce); - log.log(LogLevel.WARNING, msg); - return new ErrorHttpResponse(Status.BAD_REQUEST, msg); - } catch (Exception e) { - String msg = "Could not initialize document parsing"; - log.log(LogLevel.WARNING, "Could not initialize document parsing", e); - return new ErrorHttpResponse(Status.INTERNAL_SERVER_ERROR, msg + ": " + Exceptions.toMessageString(e)); - } - - try { - feeder.waitForRequestReceived(); - } catch (InterruptedException e) { - return new ErrorHttpResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } - - return new FeedResponse(200, operations, protocolVersion.second, clientId.first); + return feedHandlerV3.handle(request); } // Protected for testing @@ -225,82 +140,6 @@ public class FeedHandler extends LoggingRequestHandler { } } - /** - * Exposed for creating mocks. - */ - protected Feeder createFeeder( - HttpRequest request, - InputStream requestInputStream, - BlockingQueue<OperationStatus> operations, - String clientId, - boolean sessionIdWasGeneratedJustNow, - int protocolVersion) throws Exception { - if (protocolVersion != 2) - throw new IllegalStateException("Protocol version " + protocolVersion + " not supported."); - - return new Feeder( - unzipStreamIfNeeded(requestInputStream, request), - new FeedReaderFactory(), - docTypeManager, - operations, - popClient(clientId), - new FeederSettings(request), - clientId, - sessionIdWasGeneratedJustNow, - sourceSessionParams(request), - sessionCache, - this, - metric, - feedReplyHandler, - localHostname); - } - - private Tuple2<String, Boolean> sessionId(HttpRequest request) { - boolean sessionIdWasGeneratedJustNow = false; - String sessionId = request.getHeader(Headers.SESSION_ID); - if (sessionId == null) { - sessionId = Long.toString(this.sessionId.incrementAndGet()) + "-" + - remoteHostAddressAndPort(request.getJDiscRequest()) + "#" + - localHostname; - sessionIdWasGeneratedJustNow = true; - } - return new Tuple2<>(sessionId, sessionIdWasGeneratedJustNow); - } - - private static String remoteHostAddressAndPort(com.yahoo.jdisc.http.HttpRequest httpRequest) { - SocketAddress remoteAddress = httpRequest.getRemoteAddress(); - if (remoteAddress instanceof InetSocketAddress) { - InetSocketAddress isa = (InetSocketAddress) remoteAddress; - return isa.getAddress().getHostAddress() + "-" + isa.getPort(); - } - return ""; - } - - private static String resolveLocalHostname() { - String hostname = HostName.getLocalhost(); - if (hostname.equals("localhost")) { - return ""; - } - return hostname; - } - - /** - * Exposed for use when creating mocks. - */ - protected SourceSessionParams sourceSessionParams(HttpRequest request) { - SourceSessionParams params = new SourceSessionParams(); - String timeout = request.getHeader(Headers.TIMEOUT); - - if (timeout != null) { - try { - params.setTimeout(Double.parseDouble(timeout)); - } catch (NumberFormatException e) { - // NOP - } - } - return params; - } - @Override protected void destroy() { feedHandlerV3.destroy(); @@ -316,33 +155,5 @@ public class FeedHandler extends LoggingRequestHandler { private void internalDestroy() { super.destroy(); - workers.shutdown(); - cron.shutdown(); - synchronized (clients) { - for (ClientState client : clients.values()) { - client.sourceSession.getReference().close(); - } - clients.clear(); - } - } - - void putClient(final String sessionId, final ClientState value) { - synchronized (clients) { - clients.put(sessionId, value); - } } - - ClientState popClient(String sessionId) { - synchronized (clients) { - return clients.remove(sessionId); - } - } - - /** - * Guess what, testing only. - */ - void forceRunCleanClients() { - new CleanClients().run(); - } - } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java deleted file mode 100644 index f7890db3b35..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java +++ /dev/null @@ -1,537 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.server; - -import com.yahoo.collections.Tuple2; -import com.yahoo.container.jdisc.messagebus.SessionCache; -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.DocumentRemove; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; -import com.yahoo.documentapi.metrics.DocumentOperationType; -import com.yahoo.jdisc.Metric; -import com.yahoo.jdisc.ReferencedResource; -import com.yahoo.log.LogLevel; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.ReplyHandler; -import com.yahoo.messagebus.Result; -import com.yahoo.messagebus.SourceSessionParams; -import com.yahoo.messagebus.routing.ErrorDirective; -import com.yahoo.messagebus.routing.Hop; -import com.yahoo.messagebus.routing.Route; -import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.yolean.Exceptions; -import com.yahoo.text.Utf8String; -import com.yahoo.vespa.http.client.core.Encoder; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.OperationStatus; -import com.yahoo.vespa.http.server.util.ByteLimitedInputStream; -import com.yahoo.vespaxmlparser.FeedReader; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -import static com.yahoo.messagebus.ErrorCode.SEND_QUEUE_FULL; - -/** - * Read documents from client, and send them through message bus. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class Feeder implements Runnable { - - protected static final Logger log = Logger.getLogger(Feeder.class.getName()); - - final InputStream requestInputStream; - final DocumentTypeManager docTypeManager; - final BlockingQueue<OperationStatus> operations; - final BlockingQueue<OperationStatus> feedReplies; - int numPending; - final FeederSettings settings; - final String clientId; - final ReferencedResource<SharedSourceSession> sourceSession; - final FeedHandler handler; - final Metric metric; - final Metric.Context metricContext; - private long prevOpsPerSecTime; // previous measurement time of OPS - private double operationsForOpsPerSec; - private final ReplyHandler feedReplyHandler; - protected final static String EOF = "End of stream"; - protected final boolean sessionIdWasGeneratedJustNow; - private final CountDownLatch requestReceived = new CountDownLatch(1); - private final FeedReaderFactory feedReaderFactory; - - public Feeder(InputStream requestInputStream, - FeedReaderFactory feedReaderFactory, - DocumentTypeManager docTypeManager, - BlockingQueue<OperationStatus> operations, - ClientState storedState, - FeederSettings settings, - String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, - SessionCache sessionCache, - FeedHandler handler, Metric metric, ReplyHandler feedReplyHandler, - String localHostname) throws Exception { - super(); - this.feedReaderFactory = feedReaderFactory; - if (storedState == null) { - if (!sessionIdWasGeneratedJustNow) { - // We do not have a stored state, BUT the session ID came in with the request. - // Possible session timeout, server restart, server reconfig, or VIP usage. Examine. - examineClientId(clientId, localHostname); - } - numPending = 0; - feedReplies = new LinkedBlockingQueue<>(); - sourceSession = retainSession(sessionParams, sessionCache); - metricContext = createClientMetricContext(metric, clientId); - prevOpsPerSecTime = System.currentTimeMillis(); - operationsForOpsPerSec = 0.0; - } else { - //we have a stored state, and the session ID was obviously not generated now. All OK. - numPending = storedState.pending; - feedReplies = storedState.feedReplies; - sourceSession = storedState.sourceSession; - metricContext = storedState.metricContext; - prevOpsPerSecTime = storedState.prevOpsPerSecTime; - operationsForOpsPerSec = storedState.operationsForOpsPerSec; - } - this.clientId = clientId; - this.sessionIdWasGeneratedJustNow = sessionIdWasGeneratedJustNow; - this.requestInputStream = requestInputStream; - this.docTypeManager = docTypeManager; - this.operations = operations; - this.settings = settings; - this.handler = handler; - this.metric = metric; - this.feedReplyHandler = feedReplyHandler; - } - protected void examineClientId(String clientId, String localHostname) { - if (!clientId.contains("#")) { - throw new UnknownClientException("Got request from client with id '" + clientId + - "', but found no session for this client. " + - "This is expected during upgrades of gateways and infrastructure nodes."); - } - int hashPos = clientId.indexOf("#"); - String supposedHostname = clientId.substring(hashPos + 1, clientId.length()); - if (supposedHostname.isEmpty()) { - throw new UnknownClientException("Got request from client with id '" + clientId + - "', but found no session for this client. Possible session " + - "timeout due to inactivity, server restart or reconfig, " + - "or bad VIP usage. " + - "This is expected during upgrades of gateways and infrastructure nodes."); - } - - if (!supposedHostname.equals(localHostname)) { - throw new UnknownClientException("Got request from client with id '" + clientId + - "', but found no session for this client. " + - "Session was originally established towards host " + - supposedHostname + ", but our hostname is " + - localHostname + ". " + - "If using VIP rotation, this could be due to a session was rotated from one server to another. " + - "Configure VIP with persistence=enabled. " + - "This is expected during upgrades of gateways and infrastructure nodes."); - } - log.log(LogLevel.DEBUG, "Client '" + clientId + "' reconnected after session inactivity, or server restart " + - "or reconfig. Re-establishing session."); - } - - - - private static Metric.Context createClientMetricContext(Metric metric, String clientId) { - // No real value in separate metric dimensions per client. - return null; - } - - /** - * Exposed for creating mocks. - */ - protected ReferencedResource<SharedSourceSession> retainSession( - SourceSessionParams sessionParams, SessionCache sessionCache) { - return sessionCache.retainSource(sessionParams); - } - - @Override - public void run() { - try { - if (handshake()) { - return; //will putClient in finally block below - } - flushResponseQueue(); - feed(); - requestReceived.countDown(); - drain(); - } catch (InterruptedException e) { - // NOP, just terminate - } catch (Exception e) { - log.log(LogLevel.WARNING, "Unhandled exception while feeding: " - + Exceptions.toMessageString(e), e); - } catch (Throwable e) { - log.log(LogLevel.WARNING, "Unhandled error while feeding: " - + Exceptions.toMessageString(e), e); - throw e; - } finally { - requestReceived.countDown(); - putClient(); - try { - enqueue("-", "-", ErrorCode.END_OF_FEED, false, null); - } catch (InterruptedException e) { - // NOP, we are already exiting the thread - } - } - } - - protected boolean handshake() throws IOException { - if (sessionIdWasGeneratedJustNow) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Handshake completed for client " + clientId + "."); - } - requestInputStream.close(); - return true; - } - return false; - } - - void feed() throws InterruptedException { - while (true) { - Result result; - String operationId; - try { - operationId = getNextOperationId(); - } catch (IOException ioe) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, Exceptions.toMessageString(ioe), ioe); - } - break; - } - - //noinspection StringEquality - if (operationId == EOF) { - break; - } - - Tuple2<String, Message> msg; - try { - msg = getNextMessage(operationId); - setRoute(msg); - } catch (Exception e) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, Exceptions.toMessageString(e), e); - } - //noinspection StringEquality - if (operationId != null) { //v1 always returns null, all others return something useful, or throw an exception above - msg = newErrorMessage(operationId, e); - } else { - break; - } - } - - if (msg == null) { - break; - } - - setMessageParameters(msg); - - while (true) { - try { - msg.second.pushHandler(feedReplyHandler); - if (settings.denyIfBusy) { - result = sourceSession.getResource().sendMessage(msg.second); - } else { - result = sourceSession.getResource().sendMessageBlocking(msg.second); - } - } catch (RuntimeException e) { - enqueue(msg.first, Exceptions.toMessageString(e), - ErrorCode.ERROR, false, msg.second); - return; - } - if (result.isAccepted() || result.getError().getCode() != SEND_QUEUE_FULL) { - break; - } - if (settings.denyIfBusy) { - break; - } else { - //This will never happen - Thread.sleep(100); - } - } - - if (result.isAccepted()) { - ++numPending; - updateMetrics(msg.second); - updateOpsPerSec(); - log(LogLevel.DEBUG, "Sent message successfully, document id: ", msg.first); - } else if (!result.getError().isFatal()) { - enqueue(msg.first, result.getError().getMessage(), ErrorCode.TRANSIENT_ERROR, false, msg.second); - break; - } else { - // should probably not happen, but everybody knows stuff that - // shouldn't happen, happens all the time - boolean isConditionNotMet = result.getError().getCode() == DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED; - enqueue(msg.first, result.getError().getMessage(), ErrorCode.ERROR, isConditionNotMet, msg.second); - break; - } - } - } - - private Tuple2<String, Message> newErrorMessage(String operationId, Exception e) { - Message m = new FeedErrorMessage(operationId); - Tuple2<String, Message> msg = new Tuple2<>(operationId, m); - Hop hop = new Hop(); - hop.addDirective(new ErrorDirective(Exceptions.toMessageString(e))); - Route route = new Route(); - route.addHop(hop); - m.setRoute(route); - return msg; - } - - private void updateMetrics(Message m) { - metric.set( - MetricNames.PENDING, - Double.valueOf(sourceSession.getResource().session().getPendingCount()), - null); - - metric.add(MetricNames.NUM_OPERATIONS, 1, metricContext); - - if (m instanceof PutDocumentMessage) { - metric.add(MetricNames.NUM_PUTS, 1, metricContext); - } else if (m instanceof RemoveDocumentMessage) { - metric.add(MetricNames.NUM_REMOVES, 1, metricContext); - } else if (m instanceof UpdateDocumentMessage) { - metric.add(MetricNames.NUM_UPDATES, 1, metricContext); - } - } - - private void updateOpsPerSec() { - long now = System.currentTimeMillis(); - if ((now - prevOpsPerSecTime) >= 1000) { //every second - double ms = (double) (now - prevOpsPerSecTime); - final double opsPerSec = operationsForOpsPerSec / (ms / 1000); - metric.set(MetricNames.OPERATIONS_PER_SEC, opsPerSec, metricContext); - operationsForOpsPerSec = 1.0d; - prevOpsPerSecTime = now; - } else { - operationsForOpsPerSec += 1.0d; - } - } - - private Tuple2<String, Message> getNextMessage(String operationId) throws Exception { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - Tuple2<String, Message> msg; - - getNextOperation(op); - - switch (op.getType()) { - case DOCUMENT: - msg = newPutMessage(op, operationId); - break; - case REMOVE: - msg = newRemoveMessage(op, operationId); - break; - case UPDATE: - msg = newUpdateMessage(op, operationId); - break; - default: - // typical end of feed - return null; - } - log(LogLevel.DEBUG, "Successfully deserialized document id: ", msg.first); - return msg; - } - - private void setMessageParameters(Tuple2<String, Message> msg) { - msg.second.setContext(new ReplyContext(msg.first, feedReplies, DocumentOperationType.fromMessage(msg.second))); - if (settings.traceLevel != null) { - msg.second.getTrace().setLevel(settings.traceLevel); - } - if (settings.priority != null) { - try { - DocumentProtocol.Priority priority = DocumentProtocol.Priority.valueOf(settings.priority); - if (msg.second instanceof DocumentMessage) { - ((DocumentMessage) msg.second).setPriority(priority); - } - } - catch (IllegalArgumentException i) { - log.severe(i.getMessage()); - } - } - } - - private void setRoute(Tuple2<String, Message> msg) { - if (settings.route != null) { - msg.second.setRoute(settings.route); - } - } - - protected void getNextOperation(VespaXMLFeedReader.Operation op) throws Exception { - int length = readByteLength(); - - try (InputStream limitedInputStream = new ByteLimitedInputStream(requestInputStream, length)){ - FeedReader reader = feedReaderFactory.createReader(limitedInputStream, docTypeManager, settings.dataFormat); - reader.read(op); - } - } - - protected String getNextOperationId() throws IOException { - return readOperationId(); - } - - private String readOperationId() throws IOException { - StringBuilder idBuf = new StringBuilder(100); - int c; - while ((c = requestInputStream.read()) != -1) { - if (c == 32) { - break; - } - idBuf.append((char) c); //it's ASCII - } - if (idBuf.length() == 0) { - return EOF; - } - return Encoder.decode(idBuf.toString(), new StringBuilder(idBuf.length())).toString(); - } - - private int readByteLength() throws IOException { - StringBuilder lenBuf = new StringBuilder(8); - int c; - while ((c = requestInputStream.read()) != -1) { - if (c == 10) { - break; - } - lenBuf.append((char) c); //it's ASCII - } - if (lenBuf.length() == 0) { - throw new IllegalStateException("Operation length missing."); - } - return Integer.valueOf(lenBuf.toString(), 16); - } - - protected final void log(LogLevel level, Object... msgParts) { - StringBuilder s; - - if (!log.isLoggable(level)) { - return; - } - - s = new StringBuilder(); - for (Object part : msgParts) { - s.append(part.toString()); - } - - log.log(level, s.toString()); - } - - private Tuple2<String, Message> newUpdateMessage(Operation op, String operationId) { - DocumentUpdate update = op.getDocumentUpdate(); - update.setCondition(op.getCondition()); - Message msg = new UpdateDocumentMessage(update); - - String id = (operationId == null) ? update.getId().toString() : operationId; - return new Tuple2<>(id, msg); - } - - private Tuple2<String, Message> newRemoveMessage(Operation op, String operationId) { - DocumentRemove remove = new DocumentRemove(op.getRemove()); - remove.setCondition(op.getCondition()); - Message msg = new RemoveDocumentMessage(remove); - - String id = (operationId == null) ? remove.getId().toString() : operationId; - return new Tuple2<>(id, msg); - } - - private Tuple2<String, Message> newPutMessage(Operation op, String operationId) { - DocumentPut put = new DocumentPut(op.getDocument()); - put.setCondition(op.getCondition()); - Message msg = new PutDocumentMessage(put); - - String id = (operationId == null) ? put.getId().toString() : operationId; - return new Tuple2<>(id, msg); - } - - - void flushResponseQueue() throws InterruptedException { - OperationStatus status = feedReplies.poll(); - while (status != null) { - decreasePending(status); - status = feedReplies.poll(); - } - } - - void putClient() { - handler.putClient(clientId, - new ClientState(numPending, - feedReplies, sourceSession, metricContext, - prevOpsPerSecTime, operationsForOpsPerSec)); - } - - void drain() throws InterruptedException { - if (settings.drain) { - while (numPending > 0) { - OperationStatus o = feedReplies.take(); - decreasePending(o); - } - } - } - - private void decreasePending(OperationStatus o) throws InterruptedException { - operations.put(o); - --numPending; - } - - private void enqueue(String id, String message, ErrorCode code, boolean isConditionalNotMet, Message msg) - throws InterruptedException { - String traceMessage = msg != null && msg.getTrace() != null && msg.getTrace().getLevel() > 0 - ? msg.getTrace().toString() - : ""; - operations.put(new OperationStatus(message, id, code, isConditionalNotMet, traceMessage)); - } - - public void waitForRequestReceived() throws InterruptedException { - requestReceived.await(1, TimeUnit.HOURS); - } - - public class FeedErrorMessage extends Message { - private long sequenceId; - - private FeedErrorMessage(String operationId) { - try { - DocumentId id = new DocumentId(operationId); - sequenceId = Arrays.hashCode(id.getGlobalId()); - } catch (Exception e) { - sequenceId = 0; - } - } - - @Override - public Utf8String getProtocol() { - return new Utf8String("vespa-feed-handler-internal-bogus-protocol"); - } - - @Override - public int getType() { - return 1234; - } - - @Override - public boolean hasSequenceId() { - return true; - } - - @Override - public long getSequenceId() { - return sequenceId; - } - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java index b4067c3fe78..6bca3637297 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java @@ -32,4 +32,5 @@ public class DocumentApiApplicationTest { socket.close(); return port; } + } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore b/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore +++ /dev/null diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java deleted file mode 100644 index c0cc907c671..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.jdisc.HeaderFields; -import com.yahoo.jdisc.Metric; -import com.yahoo.container.logging.AccessLog; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; -import com.yahoo.vespa.http.server.FeedHandler; -import com.yahoo.vespa.http.server.Feeder; -import org.junit.Test; - -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Unit test for FeedHandler class. - * - * @author dybis - */ -public class FeedHandlerTest { - - /** - * This class extends FeedHandler and allows to create a custom Feeder. - */ - static class TestFeedHandler extends FeedHandler { - private final CountDownLatch countDownLatch = new CountDownLatch(1); - - public TestFeedHandler() throws Exception { - super(new FeedHandler.Context(Executors.newCachedThreadPool(), - mock(AccessLog.class), - mock(Metric.class)), - null, null, null, MetricReceiver.nullImplementation); - } - - /** - * Builds a feeder that blocks until countDownLatch is stepped down. - */ - @Override - protected Feeder createFeeder( - com.yahoo.container.jdisc.HttpRequest request, - InputStream requestInputStream, - final BlockingQueue<OperationStatus> operations, - String clientId, - boolean sessionIdWasGeneratedJustNow, - int protocolVersion) throws Exception { - Feeder feeder = mock(Feeder.class); - doAnswer(invocation -> { - try { - countDownLatch.await(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - return null; - }).when(feeder).waitForRequestReceived(); - return feeder; - } - } - - /** - * nginx require that a post is finished before the server ack with a response. This behaviour is verified - * in this test - */ - @Test - public void testResponseIsSentAfterWaitForRequestReceivedReturns() throws Exception { - HttpRequest request = mock(HttpRequest.class); - - // Create a request with valid version. - com.yahoo.jdisc.http.HttpRequest jdiscRequest = mock(com.yahoo.jdisc.http.HttpRequest.class); - HeaderFields headerFields = mock(HeaderFields.class); - List<String> version = new ArrayList<>(); - version.add("2"); - when(headerFields.get(Headers.VERSION)).thenReturn(version); - when(jdiscRequest.headers()).thenReturn(headerFields); - when(request.getJDiscRequest()).thenReturn(jdiscRequest); - - TestFeedHandler feedHandler = new TestFeedHandler(); - // After a short period, make the feed finish. - new Thread(() -> { - try { - Thread.sleep(50); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - feedHandler.countDownLatch.countDown(); - }).start(); - // This should not return before countdown latch is stepped down. - feedHandler.handle(request); - // This should always returns after the countDownLatch has become zero. This might cause false positive, - // but not false negatives. This is fine. - assertThat(feedHandler.countDownLatch.getCount(), is(0L)); - - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java deleted file mode 100755 index 2c99ec194c3..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java +++ /dev/null @@ -1,1064 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import com.yahoo.component.ComponentId; -import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.docproc.CallStack; -import com.yahoo.jdisc.HeaderFields; -import com.yahoo.messagebus.*; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.container.Container; -import com.yahoo.docproc.*; -import com.yahoo.docproc.jdisc.DocumentProcessingHandler; -import com.yahoo.docproc.jdisc.DocumentProcessingHandlerParameters; -import com.yahoo.document.*; -import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; -import com.yahoo.documentapi.messagebus.protocol.*; -import com.yahoo.feedapi.DummySessionFactory; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.jdisc.handler.RequestHandler; -import com.yahoo.messagebus.routing.Route; -import com.yahoo.vespaclient.ClusterDef; -import com.yahoo.vespaclient.ClusterList; -import com.yahoo.vespaclient.config.FeederConfig; -import org.junit.After; -import org.junit.Test; - -import java.io.*; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; -import java.util.logging.Logger; -import java.util.zip.GZIPOutputStream; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@SuppressWarnings("deprecation") // VespaFeedHandler classes are going away on Vespa 7 -public class VespaFeedHandlerTestCase { - - private VespaFeedHandler feedHandler; - private VespaFeedHandlerRemove removeHandler; - private VespaFeedHandlerStatus statusHandler; - private VespaFeedHandlerRemoveLocation removeLocationHandler; - private FeedContext context; - - private DummySessionFactory factory; - private final String xmlFilesPath = "src/test/files/feedhandler/"; - - public void setup(com.yahoo.messagebus.Error e, LoadTypeConfig loadTypeCfg, - boolean autoReply, - DummySessionFactory.ReplyFactory autoReplyFactory) throws Exception { - DocumentTypeManager docMan = new DocumentTypeManager(); - DocumentTypeManagerConfigurer.configure(docMan, "file:" + xmlFilesPath + "documentmanager.cfg"); - - if (autoReply) { - if (autoReplyFactory != null) { - factory = DummySessionFactory.createWithAutoReplyFactory(autoReplyFactory); - } else { - factory = DummySessionFactory.createWithErrorAutoReply(e); - } - } else { - factory = DummySessionFactory.createDefault(); - } - - context = new FeedContext(new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder()), loadTypeCfg), factory, docMan, new ClusterList(), new NullFeedMetric(true)); - - Executor threadPool = Executors.newCachedThreadPool(); - feedHandler = new VespaFeedHandler(context, threadPool); - removeHandler = new VespaFeedHandlerRemove(context, threadPool); - statusHandler = new VespaFeedHandlerStatus(context, false, false, threadPool); - removeLocationHandler = new VespaFeedHandlerRemoveLocation(context, threadPool); - - CallStack dpCallstack = new CallStack("bar"); - dpCallstack.addLast(new TestDocProc()); - dpCallstack.addLast(new TestLaterDocProc()); - - DocprocService myservice = new DocprocService("bar"); - myservice.setCallStack(dpCallstack); - myservice.setInService(true); - - ComponentRegistry<DocprocService> registry = new ComponentRegistry<DocprocService>(); - registry.register(new ComponentId(myservice.getName()), myservice); - - DocumentProcessingHandler handler = new DocumentProcessingHandler(registry, - new ComponentRegistry<>(), new ComponentRegistry<>(), - new DocumentProcessingHandlerParameters()); - - Container container = Container.get(); - ComponentRegistry<RequestHandler> requestHandlerComponentRegistry = new ComponentRegistry<>(); - requestHandlerComponentRegistry.register(new ComponentId(DocumentProcessingHandler.class.getName()), handler); - container.setRequestHandlerRegistry(requestHandlerComponentRegistry); - } - - public void setup(com.yahoo.messagebus.Error e) throws Exception { - setup(e, new LoadTypeConfig(new LoadTypeConfig.Builder()), true, null); - } - - public void setupWithReplyFactory(DummySessionFactory.ReplyFactory autoReplyFactory) throws Exception { - setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder()), true, autoReplyFactory); - } - - public void setup() throws Exception { - setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder()), false, null); - } - - @After - public void resetContainer() { - Container.resetInstance(); - } - - - @Test - public void testLoadTypes() throws Exception { - List<LoadTypeConfig.Type.Builder> typeBuilder = new ArrayList<>(); - typeBuilder.add(new LoadTypeConfig.Type.Builder().id(1234).name("foo").priority("VERY_LOW")); - typeBuilder.add(new LoadTypeConfig.Type.Builder().id(4567).name("bar").priority("NORMAL_3")); - - setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder().type(typeBuilder)), true, null); - - { - Result res = testRequest(HttpRequest.createTestRequest("remove?id=doc:test:removeme&loadtype=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:removeme", d.toString()); - assertEquals(new LoadType(1234, "foo", DocumentProtocol.Priority.VERY_LOW), ((DocumentMessage)m).getLoadType()); - assertEquals(DocumentProtocol.Priority.VERY_LOW, ((DocumentMessage)m).getPriority()); - } - } - - @Test - public void testPostXML() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - assertTrue(res.output.contains("count=\"2\"")); - assertTrue(res.error == null); - } - - @Test - public void testPostXMLAsync() throws Exception { - setup(); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?asynchronous=true"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - // Should not have metrics at this point. - assertTrue(!res.output.contains("count=\"2\"")); - assertTrue(res.error == null); - } - - - @Test - public void testPostGZIPedXML() throws Exception { - setup(null); - Result res = testFeedGZIP(xmlFilesPath + "test10b.xml", "feed?"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - assertTrue(res.error == null); - } - - @Test - public void testDocProc() throws Exception { - setup(null); - - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - Document d = ((PutDocumentMessage)m).getDocumentPut().getDocument(); - - assertEquals("doc:news:http://news10a", d.getId().toString()); - assertEquals(new IntegerFieldValue(1234), d.getFieldValue("last_downloaded")); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - Document d = ((PutDocumentMessage)m).getDocumentPut().getDocument(); - - assertEquals("doc:news:http://news10b", d.getId().toString()); - assertEquals(new IntegerFieldValue(1234), d.getFieldValue("last_downloaded")); - } - } - - @Test - public void testPostXMLVariousTypes() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10.xml", "feed?"); - - assertEquals(5, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - { - Message m = res.messages.get(2); - assertEquals(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, m.getType()); - DocumentId d = ((UpdateDocumentMessage)m).getDocumentUpdate().getId(); - assertEquals("doc:news:http://news10c", d.toString()); - } - { - Message m = res.messages.get(3); - assertEquals(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, m.getType()); - DocumentId d = ((UpdateDocumentMessage)m).getDocumentUpdate().getId(); - assertEquals("doc:news:http://news10d", d.toString()); - } - { - Message m = res.messages.get(4); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:news:http://news10e", d.toString()); - } - - String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); - - assertEquals("<result>\n" + - "\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <latency count=\"5\"/>\n" + - " <count count=\"5\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"1\"/>\n" + - " <count count=\"1\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - "\n" + - "</result>\n", val); - } - - @Test - public void testStatusPage() throws Exception { - setup(null); - - testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); - testFeed(xmlFilesPath + "test10.xml", "feed?"); - testFeed(xmlFilesPath + "test10.xml", "feed?route=storage"); - testFeed(xmlFilesPath + "test_removes", "remove?"); - - assertEquals(2, factory.sessionsCreated()); - Result res = testRequest(HttpRequest.createTestRequest("feedstatus?", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - - String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); - val = val.replaceAll("to=\"[0-9]*\"", "to=\"0\""); - - assertEquals("<status>\n" + - "\n" + - " <snapshot name=\"Total metrics from start until current time\" from=\"0\" to=\"0\" period=\"0\">\n" + - " <routes>\n" + - " <route name=\"total\">\n" + - " <total>\n" + - " <latency count=\"14\"/>\n" + - " <count count=\"14\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"6\"/>\n" + - " <count count=\"6\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <latency count=\"9\"/>\n" + - " <count count=\"9\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"3\"/>\n" + - " <count count=\"3\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " <route name=\"storage\">\n" + - " <total>\n" + - " <latency count=\"5\"/>\n" + - " <count count=\"5\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"1\"/>\n" + - " <count count=\"1\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " </routes>\n" + - " </snapshot>\n" + - "\n" + - "</status>\n", val); - } - - @Test - public void testStatusPage2() throws Exception { - setup(null); - - testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); - testFeed(xmlFilesPath + "test10.xml", "feed?"); - testFeed(xmlFilesPath + "test10.xml", "feed?route=storage"); - testFeed(xmlFilesPath + "test_removes", "remove?"); - - assertEquals(2, factory.sessionsCreated()); - Result res = testRequest(HttpRequest.createTestRequest("feed?status", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - - String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); - val = val.replaceAll("to=\"[0-9]*\"", "to=\"0\""); - - assertEquals("<status>\n" + - "\n" + - " <routes>\n" + - " <route name=\"total\" description=\"Messages sent to all routes\">\n" + - " <total description=\"All kinds of messages sent to the given route\">\n" + - " <latency count=\"14\"/>\n" + - " <count count=\"14\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"6\"/>\n" + - " <count count=\"6\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " <route name=\"default\" description=\"Messages sent to the named route\">\n" + - " <total description=\"All kinds of messages sent to the given route\">\n" + - " <latency count=\"9\"/>\n" + - " <count count=\"9\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"3\"/>\n" + - " <count count=\"3\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " <route name=\"storage\" description=\"Messages sent to the named route\">\n" + - " <total description=\"All kinds of messages sent to the given route\">\n" + - " <latency count=\"5\"/>\n" + - " <count count=\"5\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"1\"/>\n" + - " <count count=\"1\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " </routes>\n" + - "\n" + - "</status>\n", val); - } - - @Test - public void testMetricForIgnoredDocumentsIsIncreased() throws Exception { - DummySessionFactory.ReplyFactory replyFactory = new DummySessionFactory.ReplyFactory() { - @Override - public Reply createReply(Message m) { - return new DocumentIgnoredReply(); - } - }; - setupWithReplyFactory(replyFactory); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); - assertEquals(2, res.messages.size()); - - String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); - - assertEquals("<result>\n" + - "\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <ignored count=\"2\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <ignored count=\"2\"/>\n" + - " </putdocument>\n" + - " </route>\n" + - "\n" + - "</result>\n", val); - } - - @Test - public void testPostXMLWithMBusFailureAllowed() throws Exception { - setup(new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_BUCKET_DELETED, "Hello world in <document>")); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?abortonfeederror=false"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - String val = res.output.replaceAll("average=\"[0-9]*\" last=\"[0-9]*\" min=\"[0-9]*\" max=\"[0-9]*\" ", ""); - System.out.println(val); - - assertEquals("<result>\n" + - "\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <errors>\n" + - " <error name=\"total\" count=\"2\"/>\n" + - " <error name=\"BUCKET_DELETED\" count=\"2\"/>\n" + - " </errors>\n" + - " </total>\n" + - " <putdocument>\n" + - " <errors>\n" + - " <error name=\"total\" count=\"2\"/>\n" + - " <error name=\"BUCKET_DELETED\" count=\"2\"/>\n" + - " </errors>\n" + - " </putdocument>\n" + - " </route>\n\n" + - " <errors count=\"2\">\n" + - " <error message=\"PUT[doc:news:http://news10a] [BUCKET_DELETED] Hello world in <document>\"/>\n" + - " <error message=\"PUT[doc:news:http://news10b] [BUCKET_DELETED] Hello world in <document>\"/>\n" + - " </errors>\n" + - "\n" + - "</result>\n", val); - - assertTrue(res.error != null); - assertTrue(res.errorCount > 0); - } - - @Test - public void testPostXMLWithMBusFailure() throws Exception { - setup(new com.yahoo.messagebus.Error(32, "Hello world")); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); - - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - - String val = res.output.replaceAll("average=\"[0-9]*\" last=\"[0-9]*\" min=\"[0-9]*\" max=\"[0-9]*\" ", ""); - assertEquals("<result>\n" + - "\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <errors>\n" + - " <error name=\"total\" count=\"1\"/>\n" + - " <error name=\"UNKNOWN(32)\" count=\"1\"/>\n" + - " </errors>\n" + - " </total>\n" + - " <putdocument>\n" + - " <errors>\n" + - " <error name=\"total\" count=\"1\"/>\n" + - " <error name=\"UNKNOWN(32)\" count=\"1\"/>\n" + - " </errors>\n" + - " </putdocument>\n" + - " </route>\n\n" + - " <errors count=\"1\">\n" + - " <error message=\"PUT[doc:news:http://news10a] [UNKNOWN(32)] Hello world\"/>\n" + - " </errors>\n" + - "\n" + - "</result>\n", val); - - assertTrue(res.error != null); - assertTrue(res.errorCount > 0); - } - - @Test - public void testPostXMLWithIllegalDocId() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_docid.xml", "feed?"); - - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - } - - @Test - public void testPostXMLWithIllegalDocIdAllowFailure() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_docid.xml", "feed?abortondocumenterror=false"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:ok", d.toString()); - } - } - - @Test - public void testPostUnparseableXML() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_xml.xml", "feed?"); - - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - } - - @Test - public void testOverrides() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222&route=storage&priority=HIGH_2&totaltimeout=-1"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertEquals(2222, m.getTimeRemaining()); - assertEquals(Route.parse("storage"), m.getRoute()); - assertEquals(DocumentProtocol.Priority.HIGH_2, ((DocumentMessage)m).getPriority()); - } - } - - @Test - public void testTimeoutWithNoUpperBound() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222&totaltimeout=-1"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertEquals(2222, m.getTimeRemaining()); - } - } - - @Test - public void testTimeout() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertTrue(2222 >= m.getTimeRemaining()); - } - } - - @Test - public void testTotalTimeout() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?totaltimeout=2.222"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertTrue(2222 >= m.getTimeRemaining()); - } - } - - @Test - public void testTotalTimeoutAndNormalTimeout() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?totaltimeout=1000&timeout=2.222"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertEquals(2222, m.getTimeRemaining()); - } - } - - @Test - public void testBogusPriority() throws Exception { - try { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2222&route=storage&priority=HIPSTER_DOOFUS"); - assertTrue(false); - } catch (IllegalArgumentException e) { - } - } - - @Test - public void testPostXMLWithIllegalDocIdFirst() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_docid_first.xml", "feed?"); - - assertEquals(0, res.messages.size()); - } - - @Test - public void testPostXMLWithIllegalDocIdFirstNoAbort() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_docid_first.xml", "feed?abortondocumenterror=false"); - - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - } - - @Test - public void testSimpleRemove() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id=doc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:removeme", d.toString()); - } - } - - @Test - public void testRemoveUser() throws Exception { - setup(null); - - context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?user=1234", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); - String selection = ((RemoveLocationMessage)m).getDocumentSelection(); - assertEquals("storage", m.getRoute().toString()); - assertEquals("id.user=1234", selection); - } - } - - @Test - public void testRemoveGroup() throws Exception { - setup(null); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); - String selection = ((RemoveLocationMessage)m).getDocumentSelection(); - assertEquals("storage", m.getRoute().toString()); - assertEquals("id.group=\"foo\"", selection); - } - } - - @Test - public void testRemoveBadSyntax() throws Exception { - setup(null); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo&user=12345", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(0, res.messages.size()); - assertTrue(res.error.toString().contains("Exactly one of")); - } - - @Test - public void testRemoveGroupMultipleClusters() throws Exception { - setup(null); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage1", "storage/cluster.storage1")); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage2", "storage/cluster.storage2")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(0, res.messages.size()); - assertTrue(res.error.toString().contains("More than one")); - } - - @Test - public void testRemoveGroupNoClusters() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(0, res.messages.size()); - assertTrue(res.error.toString().contains("No storage clusters")); - } - - @Test - public void testRemoveSelection() throws Exception { - setup(null); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?selection=id.user=1234", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); - String selection = ((RemoveLocationMessage)m).getDocumentSelection(); - assertEquals("id.user=1234", selection); - } - } - - @Test - public void testSimpleRemoveIndex() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=doc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:removeme", d.toString()); - } - } - - @Test - public void testPostRemove() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_removes", "remove?"); - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:remove1", d.toString()); - } - - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:remove2", d.toString()); - } - } - - @Test - public void testRemoveBogusId() throws Exception { - try { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id=unknowndoc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertTrue(false); - } catch (Exception e) { - } - } - - @Test - public void testMultiRemove() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=doc:test:removeme&id[1]=doc:test:remove2&id[2]=doc:test:remove3", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(3, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:removeme", d.toString()); - } - - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:remove2", d.toString()); - } - - { - Message m = res.messages.get(2); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:remove3", d.toString()); - } - } - - @Test - public void testMultiRemoveSameDoc() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=userdoc:footype:1234:foo&id[1]=userdoc:footype:1234:foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - } - - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - } - } - - @Test - public void testFeedHandlerStatusCreation() throws Exception { - VespaFeedHandlerStatus status = new VespaFeedHandlerStatus( - new FeedContext(new MessagePropertyProcessor( - new FeederConfig(new FeederConfig.Builder()), - new LoadTypeConfig(new LoadTypeConfig.Builder())), - factory, null, new ClusterList(), new NullFeedMetric(true)), - true, true, - Executors.newCachedThreadPool()); - } - - private class TestDocProc extends DocumentProcessor { - @Override - public Progress process(Processing processing) { - for (DocumentOperation op : processing.getDocumentOperations()) { - if (op instanceof DocumentPut) { - Document document = ((DocumentPut)op).getDocument(); - document.setFieldValue("last_downloaded", new IntegerFieldValue(1234)); - } - } - return Progress.DONE; - } - } - - private class TestLaterDocProc extends DocumentProcessor { - private final Logger log = Logger.getLogger(TestLaterDocProc.class.getName()); - - private int counter = 0; - @Override - public Progress process(Processing processing) { - synchronized (this) { - counter++; - if (counter % 2 == 1) { - log.info("Returning LATER."); - return Progress.LATER; - } - log.info("Returning DONE."); - return Progress.DONE; - } - } - } - - private Result testRequest(HttpRequest req) throws Exception { - HttpResponse response = null; - String feedPrefix = "feed"; - String removePrefix = "remove"; - String feedStatusPrefix = "feedstatus"; - String removeLocationPrefix = "removelocation"; - - if (req.getUri().getPath().startsWith(feedPrefix)) { - response = feedHandler.handle(req); - } - if (req.getUri().getPath().startsWith(removePrefix)) { - response = removeHandler.handle(req); - } - if (req.getUri().getPath().startsWith(feedStatusPrefix)) { - response = statusHandler.handle(req); - } - if (req.getUri().getPath().startsWith(removeLocationPrefix)) { - response = removeLocationHandler.handle(req); - } - - ByteArrayOutputStream output = new ByteArrayOutputStream(); - response.render(output); - - Result res = new Result(); - res.messages = factory.messages; - res.output = new String(output.toByteArray()); - - if (response instanceof FeedResponse) { - FeedResponse feedResponse = (FeedResponse)response; - res.error = feedResponse.getErrorMessageList().isEmpty() ? null : feedResponse.getErrorMessageList().get(0); - res.errorCount = feedResponse.getErrorMessageList().size(); - assertTrue(feedResponse.isSuccess() == (res.errorCount == 0)); - } - return res; - } - - private Result testFeed(String xmlFile, String request) throws Exception { - return testRequest(new FileRequest(new File(xmlFile), request).toRequest()); - } - - private Result testFeedGZIP(String xmlFile, String request) throws Exception { - return testRequest(new FileRequest(new File(xmlFile), request, true).toRequest()); - } - - private class FileRequest { - - private final String req; - private final File f; - private boolean gzip = false; - - FileRequest(File f, String req) { - this.req = req; - this.f = f; - } - - FileRequest(File f, String req, boolean gzip) { - this.f = f; - this.req = req; - this.gzip = gzip; - } - - public InputStream getData() { - try { - InputStream fileStream = new FileInputStream(f); - if (gzip) { - // Not exactly pretty, but in lack of an elegant way of transcoding - ByteArrayOutputStream rawOut = new ByteArrayOutputStream(); - GZIPOutputStream compressed = new GZIPOutputStream(rawOut); - byte[] buffer = new byte[1024]; - int read = -1; - while (true) { - read = fileStream.read(buffer); - if (read == -1) break; - compressed.write(buffer, 0, read); - } - compressed.finish(); - compressed.flush(); - rawOut.flush(); - return new ByteArrayInputStream(rawOut.toByteArray()); - } - return fileStream; - } catch (Exception e) { - return null; - } - } - - public void addHeaders(HeaderFields headers) { - headers.add("Content-Type", "image/jpeg"); - if (gzip) - headers.add("Content-Encoding", "gzip"); - } - - public HttpRequest toRequest() { - HttpRequest request = HttpRequest.createTestRequest(req, com.yahoo.jdisc.http.HttpRequest.Method.GET, getData()); - addHeaders(request.getJDiscRequest().headers()); - return request; - } - - } - - private class Result { - private List<Message> messages; - private String output; - private com.yahoo.processing.request.ErrorMessage error; - private int errorCount; - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java deleted file mode 100644 index b7a911109cc..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.BucketId; -import com.yahoo.documentapi.ProgressToken; -import com.yahoo.documentapi.VisitorIterator; -import org.junit.Test; - -import java.util.Set; -import java.util.TreeSet; - -import static org.junit.Assert.*; - -@SuppressWarnings("deprecation") -public class ContinuationHitTest { - - private static final String SINGLE_BUCKET_URL_SAFE_BASE64 - = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAHqNFZ4mrz-_wAAAAAAAAAA"; - private static final String MULTI_BUCKET_URL_SAFE_BASE64 - = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz--gAAAAAAAAAA6" + - "jRWeJq8_vsAAAAAAAAAAOo0VniavP7_AAAAAAAAAAA="; - - @Test - public void continuationTokensAreUrlSafeBase64Encoded() throws Exception { - ContinuationHit hit = new ContinuationHit(createSingleBucketProgress()); - // We want -_ instead of +/ - assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, hit.getValue()); - } - - @Test - public void continuationTokensAreNotBrokenIntoMultipleLines() throws Exception { - ContinuationHit hit = new ContinuationHit(createMultiBucketProgress()); - assertTrue(hit.getValue().length() > 76); // Ensure we exceed MIME line length limits. - assertFalse(hit.getValue().contains("\n")); - } - - @Test - public void decodingAcceptsUrlSafeTokens() throws Exception { - final ProgressToken token = ContinuationHit.getToken(SINGLE_BUCKET_URL_SAFE_BASE64); - // Roundtrip should yield identical results. - assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, - new ContinuationHit(token).getValue()); - } - - /** - * Legacy Base64 encoder emitted MIME Base64. Ensure we handle tokens from that era. - */ - @Test - public void decodingAcceptsLegacyNonUrlSafeTokens() throws Exception { - final String legacyBase64 = convertedToMimeBase64Chars(SINGLE_BUCKET_URL_SAFE_BASE64); - final ProgressToken legacyToken = ContinuationHit.getToken(legacyBase64); - - assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, - new ContinuationHit(legacyToken).getValue()); - } - - /** - * Legacy Base64 encoder would happily output line breaks after each MIME line - * boundary. Ensure we handle these gracefully. - */ - @Test - public void decodingAcceptsLegacyMimeLineBrokenTokens() throws Exception { - final String multiBucketLegacyToken = - "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz++gAAAAAAAAAA6jRWeJq8/vsA\r\n" + - "AAAAAAAAAOo0VniavP7/AAAAAAAAAAA="; - final ProgressToken legacyToken = ContinuationHit.getToken(multiBucketLegacyToken); - - assertEquals(MULTI_BUCKET_URL_SAFE_BASE64, - new ContinuationHit(legacyToken).getValue()); - } - - /** - * Returns a ProgressToken whose base 64 representation will be _less_ than 76 bytes (MIME line limit) - */ - private ProgressToken createSingleBucketProgress() { - ProgressToken token = new ProgressToken(16); - // Use explicit bucket set so we can better control the binary representation - // of the buckets, and thus the values written as base 64. - Set<BucketId> buckets = new TreeSet<>(); - // This particular bucket ID will contain +/ chars when output as non-URL safe base 64. - buckets.add(new BucketId(58, 0x123456789abcfeffL)); - VisitorIterator.createFromExplicitBucketSet(buckets, 16, token); // "Prime" the token. - return token; - } - - /** - * Returns a ProgressToken whose base 64 representation will be _more_ than 76 bytes (MIME line limit) - */ - private ProgressToken createMultiBucketProgress() { - ProgressToken token = new ProgressToken(16); - Set<BucketId> buckets = new TreeSet<>(); - buckets.add(new BucketId(58, 0x123456789abcfeffL)); - buckets.add(new BucketId(58, 0x123456789abcfefaL)); - buckets.add(new BucketId(58, 0x123456789abcfefbL)); - VisitorIterator.createFromExplicitBucketSet(buckets, 16, token); // "Prime" the token. - return token; - } - - private String convertedToMimeBase64Chars(String token) { - // Doesn't split on MIME line boundaries, so not fully MIME compliant. - return token.replace('-', '+').replace('_', '/'); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java deleted file mode 100755 index 2015efa4d5d..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentType; -import com.yahoo.documentapi.VisitorParameters; -import com.yahoo.documentapi.VisitorSession; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; -import com.yahoo.feedapi.DummySessionFactory; -import com.yahoo.feedapi.SendSession; -import com.yahoo.jdisc.Metric; -import com.yahoo.messagebus.*; -import com.yahoo.messagebus.Error; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.HashMap; - -/** - * Used to automatically reply with a GetDocumentReply for every received GetDocumentMessage - */ -public class DocumentSessionFactory extends DummySessionFactory { - - private DocumentType docType; - private Error error; - // Reply instances are shared between the two collections. - List<GetDocumentReply> autoReplies; - Map<DocumentId, GetDocumentReply> autoReplyLookup = new HashMap<>(); - boolean autoReply = true; - boolean nullReply = false; - private int sessionsCreated = 0; - - private class DocumentReplySession extends SendSession { - - ReplyHandler handler; - Error e; - DummySessionFactory owner; - - public DocumentReplySession(ReplyHandler handler, Error e, DummySessionFactory owner) { - this.handler = handler; - this.e = e; - this.owner = owner; - } - - protected Result onSend(Message m, boolean blockIfQueueFull) throws InterruptedException { - if (!(m instanceof GetDocumentMessage)) { - throw new IllegalArgumentException("Expected GetDocumentMessage"); - } - GetDocumentMessage gm = (GetDocumentMessage)m; - owner.messages.add(m); - if (autoReply) { - Document replyDoc; - if (!nullReply) { - replyDoc = new Document(docType, gm.getDocumentId()); - } else { - replyDoc = null; - } - Reply r = new GetDocumentReply(replyDoc); - r.setMessage(m); - r.setContext(m.getContext()); - if (e != null) { - r.addError(e); - } - handler.handleReply(r); - } else if (owner.messages.size() == autoReplies.size()) { - // Pair up all replies with their messages - for (Message msg : owner.messages) { - GetDocumentReply reply = autoReplyLookup.get(((GetDocumentMessage)msg).getDocumentId()); - reply.setMessage(msg); - reply.setContext(msg.getContext()); - if (e != null) { - reply.addError(e); - } - } - // Now send them in the correct order. Instances are shared, so source - // messages and contexts are properly set - for (Reply reply : autoReplies) { - handler.handleReply(reply); - } - } - - return Result.ACCEPTED; - } - - public void close() { - } - } - - public DocumentSessionFactory(DocumentType docType) { - this.docType = docType; - this.error = null; - } - - public DocumentSessionFactory(DocumentType docType, Error error, boolean autoReply, GetDocumentReply... autoReplies) { - this.docType = docType; - this.error = error; - this.autoReplies = Arrays.asList(autoReplies); - for (GetDocumentReply reply : autoReplies) { - this.autoReplyLookup.put(reply.getDocument().getId(), reply); - } - this.autoReply = autoReply; - } - - public boolean isNullReply() { - return nullReply; - } - - public void setNullReply(boolean nullReply) { - this.nullReply = nullReply; - } - - public int getSessionsCreated() { - return sessionsCreated; - } - - public SendSession createSendSession(ReplyHandler r, Metric metric) { - ++sessionsCreated; - return new DocumentReplySession(r, error, this); - } - - @Override - public VisitorSession createVisitorSession(VisitorParameters p) { - return new DummyVisitorSession(p, docType); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java deleted file mode 100644 index 02d26f8bd93..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.DocumentType; -import com.yahoo.documentapi.*; -import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.Trace; - -import java.util.ArrayList; -import java.util.List; - -/** - * Stub to test visitors. - */ -public class DummyVisitorSession implements VisitorSession { - - final VisitorParameters parameters; - final DocumentType documentType; - final List<Message> autoReplyMessages = new ArrayList<>(); - - DummyVisitorSession(VisitorParameters p, DocumentType documentType) { - parameters = p; - this.documentType = documentType; - p.getLocalDataHandler().setSession(this); - addDefaultReplyMessages(); - } - - @Override - public boolean isDone() { - return true; - } - - @Override - public ProgressToken getProgress() { - return new ProgressToken(12); - } - - @Override - public Trace getTrace() { - return null; - } - - public void addDocumentReply(String docId) { - Document replyDoc = new Document(documentType, docId); - autoReplyMessages.add(new PutDocumentMessage(new DocumentPut(replyDoc))); - } - - public void addRemoveReply(String docId) { - autoReplyMessages.add(new RemoveDocumentMessage(new DocumentId(docId))); - } - - public void addDefaultReplyMessages() { - addDocumentReply("userdoc:foo:1234:bar"); - if (parameters.visitRemoves()) { - addRemoveReply("userdoc:foo:1234:removed"); - } - } - - public void clearAutoReplyMessages() { - autoReplyMessages.clear(); - } - - @Override - public boolean waitUntilDone(long l) throws InterruptedException { - for (Message msg : autoReplyMessages) { - parameters.getLocalDataHandler().onMessage(msg, new AckToken(this)); - } - return true; - } - - @Override - public void ack(AckToken ackToken) { - } - - @Override - public void abort() { - } - - @Override - public VisitorResponse getNext() { - return null; - } - - @Override - public VisitorResponse getNext(int i) throws InterruptedException { - return null; - } - - @Override - public void destroy() { - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java deleted file mode 100755 index 14fe0dc857d..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java +++ /dev/null @@ -1,1093 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.component.chain.Chain; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.feedhandler.NullFeedMetric; -import com.yahoo.jdisc.HeaderFields; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.document.*; -import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.document.datatypes.Raw; -import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.routing.Route; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.vespaclient.ClusterList; -import com.yahoo.vespaclient.config.FeederConfig; - -import org.junit.Test; - -import java.io.*; -import java.nio.ByteBuffer; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.zip.GZIPOutputStream; - -import static org.junit.Assert.*; - -@SuppressWarnings("deprecation") -public class GetSearcherTestCase { - - private DocumentTypeManager docMan = null; - private DocumentType docType; - private FeederConfig defFeedCfg = new FeederConfig(new FeederConfig.Builder()); - private LoadTypeConfig defLoadTypeCfg = new LoadTypeConfig(new LoadTypeConfig.Builder()); - - @org.junit.Before - public void setUp() { - docMan = new DocumentTypeManager(); - docType = new DocumentType("kittens"); - docType.addHeaderField("name", DataType.STRING); - docType.addField("description", DataType.STRING); - docType.addField("image", DataType.STRING); - docType.addField("fluffiness", DataType.INT); - docType.addField("foo", DataType.RAW); - docMan.registerDocumentType(docType); - } - - @org.junit.After - public void tearDown() { - docMan = null; - docType = null; - } - - private void assertHits(HitGroup hits, String... wantedHits) { - assertEquals(wantedHits.length, hits.size()); - for (int i = 0; i < wantedHits.length; ++i) { - assertTrue(hits.get(i) instanceof DocumentHit); - DocumentHit hit = (DocumentHit)hits.get(i); - assertEquals(wantedHits[i], hit.getDocument().getId().toString()); - } - } - - @Test - public void testGetSingleDocumentQuery() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:1:2")); - System.out.println("HTTP request is " + result.getQuery().getHttpRequest()); - - assertEquals(1, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - } - assertEquals(1, result.hits().size()); - assertHits(result.hits(), "userdoc:kittens:1:2"); - // By default, document hit should not have its hit fields set - DocumentHit hit = (DocumentHit)result.hits().get(0); - assertEquals(0, hit.fieldKeys().size()); - } - - @Test - public void testGetMultipleDocumentsQuery() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Query query = newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - - assertEquals(2, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - } - - { - Message m = factory.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:3:4", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - } - assertEquals(2, result.hits().size()); - assertNull(result.hits().getErrorHit()); - assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4"); - assertEquals(2, query.getHits()); - } - - // Test that you can use both query string and POSTed IDs - @Test - public void testGetMultipleDocumentsQueryAndPOST() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - String data = "userdoc:kittens:5:6\nuserdoc:kittens:7:8\nuserdoc:kittens:9:10"; - MockHttpRequest request = new MockHttpRequest(data.getBytes("utf-8"), "/get/?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4"); - Query query = new Query(request.toRequest()); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - - assertEquals(5, factory.messages.size()); - assertEquals(5, result.hits().size()); - assertNull(result.hits().getErrorHit()); - assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4", - "userdoc:kittens:5:6", "userdoc:kittens:7:8", "userdoc:kittens:9:10"); - } - - @Test - public void testGetMultipleDocumentsQueryAndGZippedPOST() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - String data = "userdoc:kittens:5:6\nuserdoc:kittens:7:8\nuserdoc:kittens:9:10"; - - // Create with automatic GZIP encoding - MockHttpRequest request = new MockHttpRequest(data.getBytes("utf-8"), "/get/?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4", true); - Query query = new Query(request.toRequest()); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - - assertEquals(5, factory.messages.size()); - assertEquals(5, result.hits().size()); - assertNull(result.hits().getErrorHit()); - assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4", - "userdoc:kittens:5:6", "userdoc:kittens:7:8", "userdoc:kittens:9:10"); - } - - /* Test that a query without any ids is passed through to the next chain */ - @Test - public void testQueryPassThrough() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - HitGroup hits = new HitGroup("mock"); - hits.add(new Hit("blernsball")); - Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(hits)); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?flarn=blern")); - - assertEquals(0, factory.messages.size()); - assertEquals(1, result.hits().size()); - assertNotNull(result.hits().get("blernsball")); - } - - /* Test that a query will contain both document hits and hits from a searcher - * further down the chain, iff the searcher returns a DocumentHit. - */ - @Test - public void testQueryPassThroughAndGet() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:1234:foo")); - doc1.setFieldValue("name", new StringFieldValue("megacat")); - doc1.setFieldValue("description", new StringFieldValue("supercat")); - doc1.setFieldValue("fluffiness", new IntegerFieldValue(10000)); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1) - }; - - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - DocumentHit backendHit = new DocumentHit(new Document(docType, new DocumentId("userdoc:kittens:5678:bar")), 5); - Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(backendHit)); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?query=flarn&id=userdoc:kittens:1234:foo")); - - assertEquals(1, factory.messages.size()); - assertEquals(2, result.hits().size()); - assertNotNull(result.hits().get("userdoc:kittens:5678:bar")); - assertNotNull(result.hits().get("userdoc:kittens:1234:foo")); - } - - @Test - public void testQueryPassThroughAndGetUnknownBackendHit() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - HitGroup hits = new HitGroup("mock"); - hits.add(new Hit("blernsball")); - Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(hits)); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?flarn=blern&id=userdoc:kittens:9:aaa")); - - assertEquals(0, factory.messages.size()); - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"18\" message=\"Internal server error.: " + - "A backend searcher to com.yahoo.storage.searcher.GetSearcher returned a " + - "hit that was not an instance of com.yahoo.storage.searcher.DocumentHit. " + - "Only DocumentHit instances are supported in the backend hit result set " + - "when doing queries that contain document identifier sets recognised by the " + - "Get Searcher.\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testConfig() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(58).route("route66").retryenabled(false)), defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=doc:batman:dahnahnahnah")); - - assertEquals(1, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("doc:batman:dahnahnahnah", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(Route.parse("route66"), gdm.getRoute()); - assertFalse(gdm.getRetryEnabled()); - assertTrue(58000 >= gdm.getTimeRemaining()); - } - } - - @Test - public void testConfigChanges() throws Exception { - String config = "raw:timeout 37\nroute \"riksveg18\"\nretryenabled true"; - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(58).route("riksveg18").retryenabled(true)), - defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=doc:batman:dahnahnahnah")); - - assertEquals(1, factory.messages.size()); - assertEquals(1, factory.getSessionsCreated()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("doc:batman:dahnahnahnah", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(Route.parse("riksveg18"), gdm.getRoute()); - assertTrue(gdm.getRetryEnabled()); - assertTrue(58000 >= gdm.getTimeRemaining()); - } - - factory.messages.clear(); - - FeederConfig newConfig = new FeederConfig(new FeederConfig.Builder() - .timeout(123) - .route("e6") - .retryenabled(false) - ); - searcher.getMessagePropertyProcessor().configure(newConfig, defLoadTypeCfg); - - new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=doc:spiderman:does_whatever_a_spider_can")); - - // riksveg18 is created again, and e6 is created as well. - assertEquals(3, factory.getSessionsCreated()); - - assertEquals(1, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("doc:spiderman:does_whatever_a_spider_can", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(Route.parse("e6"), gdm.getRoute()); - assertFalse(gdm.getRetryEnabled()); - assertTrue(123000 >= gdm.getTimeRemaining()); - } - } - - @Test - public void testQueryOverridesDefaults() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&priority=LOW_2&route=highwaytohell&timeout=58")); - - assertEquals(2, factory.messages.size()); - long lastTimeout = 58000; - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); - assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); - assertTrue(lastTimeout >= gdm.getTimeRemaining()); - lastTimeout = gdm.getTimeRemaining(); - } - - { - Message m = factory.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:3:4", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); - assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); - assertTrue(lastTimeout >= gdm.getTimeRemaining()); - } - } - - @Test - public void testQueryOverridesConfig() throws Exception { - String config = "raw:timeout 458\nroute \"route66\""; - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&priority=LOW_2&route=highwaytohell&timeout=123")); - - long lastTimeout = 123000; - assertEquals(2, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); - assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); - assertTrue(lastTimeout >= gdm.getTimeRemaining()); - lastTimeout = gdm.getTimeRemaining(); - } - - { - Message m = factory.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:3:4", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); - assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); - assertTrue(lastTimeout >= gdm.getTimeRemaining()); - } - } - - @Test - public void testBadPriorityValue() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:1:2&priority=onkel_jubalon")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: No enum const" + - "ant " + - "com.yahoo.documentapi.messagebus.protocol.DocumentProtocol" + - "." + - "Priority.onkel_jubalon\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testMultiIdBadArrayIndex() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - { - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[1]=userdoc:kittens:1:2")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: query contains document ID " + - "array that is not zero-based and/or linearly increasing\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - { - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[2]=userdoc:kittens:2:3")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: query contains document ID " + - "array that is not zero-based and/or linearly increasing\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - { - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[1]=userdoc:kittens:2:3")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: query contains document ID " + - "array that is not zero-based and/or linearly increasing\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - { - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0=userdoc:kittens:1:2")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: Malformed document ID array parameter\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - } - - @Test - public void testLegacyHeadersOnly() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:1:2&headersonly=true")); - - assertEquals(1, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[header]", gdm.getFieldSet()); - } - assertEquals(1, result.hits().size()); - assertHits(result.hits(), "userdoc:kittens:1:2"); - } - - @Test - public void testFieldSet() throws Exception { - } - - @Test - public void testConsistentResultOrdering() throws Exception { - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:1:2"))), - new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:7:8"))), - new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:555:123"))) - }; - - // Use a predefined reply list to ensure messages are answered out of order - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:555:123&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:7:8")); - - assertEquals(3, factory.messages.size()); - assertEquals(3, result.hits().size()); - // Hits must be in the same order as their document IDs in the query - assertHits(result.hits(), "userdoc:kittens:555:123", "userdoc:kittens:1:2", "userdoc:kittens:7:8"); - - assertEquals(0, ((DocumentHit)result.hits().get(0)).getIndex()); - assertEquals(1, ((DocumentHit)result.hits().get(1)).getIndex()); - assertEquals(2, ((DocumentHit)result.hits().get(2)).getIndex()); - } - - @Test - public void testResultWithSingleError() throws Exception { - com.yahoo.messagebus.Error err = new com.yahoo.messagebus.Error(32, "Alas, it went poorly"); - DocumentSessionFactory factory = new DocumentSessionFactory(docType, err, true); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4")); - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"messagebus\" code=\"32\" message=\"Alas, it went poorly\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testResultWithMultipleErrors() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); - Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:99:111")); - GetDocumentReply errorReply1 = new GetDocumentReply(doc1); - errorReply1.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 had fleas.")); - GetDocumentReply errorReply2 = new GetDocumentReply(doc2); - errorReply2.addError(new com.yahoo.messagebus.Error(456, "userdoc:kittens:99:111 shredded the curtains.")); - GetDocumentReply[] replies = new GetDocumentReply[] { - errorReply1, - errorReply2 - }; - - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:99:111")); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 had fleas.\"/>\n" + - "<error type=\"messagebus\" code=\"456\" message=\"userdoc:kittens:99:111 shredded the curtains.\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testResultWithNullDocument() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, true); - factory.setNullReply(true); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:55:bad_document_id")); - // Document not found does not produce any hit at all, error or otherwise - assertNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "</result>\n", result); - } - - @Test - public void testDefaultDocumentHitRendering() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:3:4")); - doc1.setFieldValue("name", new StringFieldValue("mittens")); - doc1.setFieldValue("description", new StringFieldValue("it's a cat")); - doc1.setFieldValue("fluffiness", new IntegerFieldValue(8)); - Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:1:2")); - doc2.setFieldValue("name", new StringFieldValue("garfield")); - doc2.setFieldValue("description", - new StringFieldValue("preliminary research indicates <em>hatred</em> of mondays. caution advised")); - doc2.setFieldValue("fluffiness", new IntegerFieldValue(2)); - Document doc3 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); - GetDocumentReply errorReply = new GetDocumentReply(doc3); - errorReply.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 had fleas.")); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - new GetDocumentReply(doc2), - errorReply - }; - - // Use a predefined reply list to ensure messages are answered out of order - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result xmlResult = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:3:4")); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 had fleas.\"/>\n" + - "</errors>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:1:2\">\n" + - " <name>garfield</name>\n" + - " <description>preliminary research indicates <em>hatred</em> of mondays. caution advised</description>\n" + - " <fluffiness>2</fluffiness>\n" + - "</document>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:3:4\">\n" + - " <name>mittens</name>\n" + - " <description>it's a cat</description>\n" + - " <fluffiness>8</fluffiness>\n" + - "</document>\n" + - "</result>\n", xmlResult); - } - - @Test - public void testDocumentFieldNoContentType() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - doc1.setFieldValue("name", "derrick"); - doc1.setFieldValue("description", "kommisar katze"); - doc1.setFieldValue("fluffiness", 0); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:5:1&field=description")); - - assertNull(result.hits().getErrorHit()); - assertEquals("text/xml", result.getTemplating().getTemplates().getMimeType()); - assertEquals("UTF-8", result.getTemplating().getTemplates().getEncoding()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>kommisar katze</result>\n", result); - } - - @Test - public void testDocumentFieldEscapeXML() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - doc1.setFieldValue("name", "asfd"); - doc1.setFieldValue("description", "<script type=\"evil/madness\">horror & screams</script>"); - doc1.setFieldValue("fluffiness", 0); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:5:1&field=description")); - - assertNull(result.hits().getErrorHit()); - assertEquals("text/xml", result.getTemplating().getTemplates().getMimeType()); - assertEquals("UTF-8", result.getTemplating().getTemplates().getEncoding()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result><script type=\"evil/madness\">horror & screams</script></result>\n", result); - } - - @Test - public void testDocumentFieldRawContent() throws Exception { - byte[] contentBytes = new byte[] { 0, -128, 127 }; - - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456")); - doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes))); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1) - }; - - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:123:456&field=foo")); - - assertNull(result.hits().getErrorHit()); - assertEquals("application/octet-stream", result.getTemplating().getTemplates().getMimeType()); - - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result); - stream.flush(); - - byte[] resultBytes = stream.toByteArray(); - assertEquals(contentBytes.length, resultBytes.length); - for (int i = 0; i < resultBytes.length; ++i) { - assertEquals(contentBytes[i], resultBytes[i]); - } - } - - @Test - public void testDocumentFieldRawWithContentOverride() throws Exception { - byte[] contentBytes = new byte[] { 0, -128, 127 }; - - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456")); - doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes))); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1) - }; - - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:123:456&field=foo&contenttype=text/fancy")); - - assertNull(result.hits().getErrorHit()); - assertEquals("text/fancy", result.getTemplating().getTemplates().getMimeType()); - - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result); - stream.flush(); - - byte[] resultBytes = stream.toByteArray(); - assertEquals(contentBytes.length, resultBytes.length); - for (int i = 0; i < resultBytes.length; ++i) { - assertEquals(contentBytes[i], resultBytes[i]); - } - } - - @Test - public void testDocumentFieldWithMultipleIDs() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&field=name")); - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: Field only valid for single document id query\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testDocumentFieldNotSet() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - doc1.setFieldValue("name", "asdf"); - doc1.setFieldValue("description", "fdsafsdf"); - doc1.setFieldValue("fluffiness", 10); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:5:1&field=image")); - - assertNotNull(result.hits().getErrorHit()); - assertEquals(1, result.hits().size()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"16\" message=\"Resource not found.: " + - "Field 'image' found in document type, but had no content in userdoc:kittens:5:1\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - - @Test - public void testDocumentFieldWithDocumentNotFound() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, true); - factory.setNullReply(true); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:1:2&field=name")); - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"16\" message=\"Resource not found.: " + - "Document not found, could not return field 'name'\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testDocumentFieldNotReachableWithHeadersOnly() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - doc1.setFieldValue("name", "asdf"); - // don't set body fields - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:5:1&field=description&headersonly=true")); - - assertNotNull(result.hits().getErrorHit()); - assertEquals(1, result.hits().size()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"4\" message=\"Invalid query parameter: " + - "Field 'description' is located in document body, but headersonly " + - "prevents it from being retrieved in userdoc:kittens:5:1\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testVespaXMLTemplate() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:3:4")); - doc1.setFieldValue("name", "mittens"); - doc1.setFieldValue("description", "it's a cat"); - doc1.setFieldValue("fluffiness", 8); - Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:1:2")); - doc2.setFieldValue("name", "garfield"); - doc2.setFieldValue("description", "preliminary research indicates <em>hatred</em> of mondays. caution advised"); - doc2.setFieldValue("fluffiness", 2); - Document doc3 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); - GetDocumentReply errorReply = new GetDocumentReply(doc3); - errorReply.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 lost in a <ni!>\"shrubbery\"</ni!>")); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - new GetDocumentReply(doc2), - errorReply - }; - - // Use a predefined reply list to ensure messages are answered out of order - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:3:4")); // TODO! - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 lost in a <ni!>"shrubbery"</ni!>\"/>\n"+ - "</errors>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:1:2\">\n" + - " <name>garfield</name>\n" + - " <description>preliminary research indicates <em>hatred</em> of mondays. caution advised</description>\n" + - " <fluffiness>2</fluffiness>\n" + - "</document>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:3:4\">\n" + - " <name>mittens</name>\n" + - " <description>it's a cat</description>\n" + - " <fluffiness>8</fluffiness>\n" + - "</document>\n" + - "</result>\n", result); - } - - @Test - public void testDocumentHitWithPopulatedHitFields() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:1234:foo")); - doc1.setFieldValue("name", new StringFieldValue("megacat")); - doc1.setFieldValue("description", new StringFieldValue("supercat")); - doc1.setFieldValue("fluffiness", new IntegerFieldValue(10000)); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1) - }; - - // Use a predefined reply list to ensure messages are answered out of order - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:1234:foo&populatehitfields=true")); - assertEquals(1, result.hits().size()); - assertHits(result.hits(), "userdoc:kittens:1234:foo"); - - DocumentHit hit = (DocumentHit)result.hits().get(0); - Iterator<Map.Entry<String, Object>> iter = hit.fieldIterator(); - Set<String> fieldSet = new TreeSet<>(); - while (iter.hasNext()) { - Map.Entry<String, Object> kv = iter.next(); - StringBuilder field = new StringBuilder(); - field.append(kv.getKey()).append(" -> ").append(kv.getValue()); - fieldSet.add(field.toString()); - } - StringBuilder fields = new StringBuilder(); - for (String s : fieldSet) { - fields.append(s).append("\n"); - } - assertEquals( - "description -> supercat\n" + - "documentid -> userdoc:kittens:1234:foo\n" + - "fluffiness -> 10000\n" + - "name -> megacat\n", - fields.toString()); - } - - @Test - public void deserializationExceptionsAreHandledGracefully() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - GetDocumentReply[] replies = new GetDocumentReply[] { - new MockFailingGetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:5:1")); - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"18\" message=\"Internal server error.: " + - "Got exception of type java.lang.RuntimeException during document " + - "deserialization: epic dragon attack\"/>\n"+ - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testJsonRendererSetting() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Query query = newQuery("?id=userdoc:kittens:1:2&format=json"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - assertFalse(result.getTemplating().getTemplates() instanceof DocumentXMLTemplate); - } - - private Query newQuery(String queryString) { - return new Query(HttpRequest.createTestRequest(queryString, com.yahoo.jdisc.http.HttpRequest.Method.GET)); - } - - private Chain<Searcher> createSearcherChain(GetDocumentReply[] replies) throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - return new Chain<>(searcher); - } - - private static class MockFailingGetDocumentReply extends GetDocumentReply { - private int countdown = 2; - - private MockFailingGetDocumentReply(Document doc) { - super(doc); - } - - @Override - public Document getDocument() { - // Reason for countdown is that the test DocumentSessionFactory calls - // getDocument once internally before code can ever reach handleReply. - if (--countdown == 0) { - throw new RuntimeException("epic dragon attack"); - } - return super.getDocument(); - } - } - - private static class MockBackend extends Searcher { - private Hit hitToReturn; - - public MockBackend(Hit hitToReturn) { - this.hitToReturn = hitToReturn; - } - - @Override - public Result search(Query query, Execution execution) { - Result result = new Result(query); - result.hits().add(hitToReturn); - return result; - } - } - - private class MockHttpRequest { - - private final String req; - private byte[] data; - private boolean gzip = false; - - MockHttpRequest(byte[] data, String req) { - this.req = req; - this.data = data; - } - - MockHttpRequest(byte[] data, String req, boolean gzip) { - this.data = data; - this.req = req; - this.gzip = gzip; - } - - public InputStream getData() { - if (gzip) { - try { - ByteArrayOutputStream rawOut = new ByteArrayOutputStream(); - GZIPOutputStream compressed = new GZIPOutputStream(rawOut); - compressed.write(data, 0, data.length); - compressed.finish(); - compressed.flush(); - rawOut.flush(); - return new ByteArrayInputStream(rawOut.toByteArray()); - } catch (Exception e) { - return null; - } - } - return new ByteArrayInputStream(data); - } - - public void addHeaders(HeaderFields headers) { - headers.add("Content-Type", "text/plain;encoding=UTF-8"); - if (gzip) - headers.add("Content-Encoding", "gzip"); - } - - public com.yahoo.container.jdisc.HttpRequest toRequest() { - com.yahoo.container.jdisc.HttpRequest request = com.yahoo.container.jdisc.HttpRequest.createTestRequest(req, com.yahoo.jdisc.http.HttpRequest.Method.GET, getData()); - addHeaders(request.getJDiscRequest().headers()); - return request; - } - - } - - public static void assertRendered(String expected,Result result) throws Exception { - assertRendered(expected,result,true); - } - - public static void assertRendered(String expected,Result result,boolean checkFullEquality) throws Exception { - if (checkFullEquality) - assertEquals(expected, ResultRenderingUtil.getRendered(result)); - else - assertTrue(ResultRenderingUtil.getRendered(result).startsWith(expected)); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java deleted file mode 100644 index 7f392b0565b..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.search.Result; - -import java.io.ByteArrayOutputStream; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; - -@SuppressWarnings("deprecation") -public class ResultRenderingUtil { - - public static String getRendered(Result result) throws Exception { - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - Charset cs = Charset.forName("utf-8"); - CharsetDecoder decoder = cs.newDecoder(); - com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result); - stream.flush(); - return decoder.decode(ByteBuffer.wrap(stream.toByteArray())).toString(); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java deleted file mode 100644 index e49d2ca4db3..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.component.chain.Chain; -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.protect.Error; -import com.yahoo.documentapi.VisitorSession; -import com.yahoo.feedhandler.NullFeedMetric; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.document.DataType; -import com.yahoo.document.DocumentType; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.documentapi.VisitorParameters; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.messagebus.StaticThrottlePolicy; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.vdslib.VisitorOrdering; -import com.yahoo.vespaclient.ClusterList; -import com.yahoo.vespaclient.config.FeederConfig; - -import org.junit.Test; - -import java.util.Arrays; - -import static org.junit.Assert.*; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@SuppressWarnings("deprecation") -public class VisitorSearcherTestCase { - - private DocumentTypeManager docMan = null; - private DocumentType docType; - DocumentSessionFactory factory; - - @org.junit.Before - public void setUp() { - docMan = new DocumentTypeManager(); - docType = new DocumentType("kittens"); - docType.addHeaderField("name", DataType.STRING); - docType.addField("description", DataType.STRING); - docType.addField("image", DataType.RAW); - docType.addField("fluffiness", DataType.INT); - docType.addField("foo", DataType.RAW); - docMan.registerDocumentType(docType); - factory = new DocumentSessionFactory(docType); - } - - public VisitSearcher create() throws Exception { - ClusterListConfig.Storage.Builder storageCluster = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foobar").name("foobar"); - ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(storageCluster)); - ClusterList clusterList = new ClusterList(clusterListCfg); - return new VisitSearcher(new FeedContext( - new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(458).route("riksveg18").retryenabled(true)), - new LoadTypeConfig(new LoadTypeConfig.Builder())), - factory, docMan, clusterList, new NullFeedMetric(true))); - } - - @Test - public void testQueryParameters() throws Exception { - VisitSearcher searcher = create(); - VisitorParameters params = searcher.getVisitorParameters( - newQuery("visit?visit.selection=id.user=1234&visit.cluster=foobar" + - "&visit.dataHandler=othercluster&visit.fieldSet=[header]&visit.fromTimestamp=112&visit.toTimestamp=224" + - "&visit.maxBucketsPerVisitor=2&visit.maxPendingMessagesPerVisitor=7&visit.maxPendingVisitors=14" + - "&visit.ordering=ASCENDING&priority=NORMAL_1&tracelevel=7&visit.visitInconsistentBuckets&visit.visitRemoves"), null); - - assertEquals("id.user=1234", params.getDocumentSelection()); - assertEquals(7, params.getMaxPending()); - assertEquals(2, params.getMaxBucketsPerVisitor()); - assertEquals(14, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount()); - assertEquals("[Storage:cluster=foobar;clusterconfigid=storage/cluster.foobar]", params.getRoute().toString()); - assertEquals("othercluster", params.getRemoteDataHandler()); - assertEquals("[header]", params.fieldSet()); - assertEquals(112, params.getFromTimestamp()); - assertEquals(224, params.getToTimestamp()); - assertEquals(VisitorOrdering.ASCENDING, params.getVisitorOrdering()); - assertEquals(DocumentProtocol.Priority.NORMAL_1, params.getPriority()); - assertEquals(7, params.getTraceLevel()); - assertEquals(true, params.visitInconsistentBuckets()); - assertEquals(true, params.visitRemoves()); - } - - @Test - public void timestampQueryParametersAreParsedAsLongs() throws Exception { - VisitorParameters params = create().getVisitorParameters( - newQuery("visit?visit.selection=id.user=1234&" + - "visit.fromTimestamp=1419021596000000&" + - "visit.toTimestamp=1419021597000000"), null); - assertEquals(1419021596000000L, params.getFromTimestamp()); - assertEquals(1419021597000000L, params.getToTimestamp()); - } - - @Test - public void testQueryParametersDefaults() throws Exception { - VisitSearcher searcher = create(); - VisitorParameters params = searcher.getVisitorParameters( - newQuery("visit?visit.selection=id.user=1234&hits=100"), null); - - assertEquals("id.user=1234", params.getDocumentSelection()); - assertEquals(1, params.getMaxBucketsPerVisitor()); - assertEquals(1, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount()); - assertEquals(1, params.getMaxFirstPassHits()); - assertEquals(1, params.getMaxTotalHits()); - assertEquals(32, params.getMaxPending()); - assertEquals(false, params.visitInconsistentBuckets()); - } - - @Test - public void testWrongCluster() throws Exception { - VisitSearcher searcher = create(); - - try { - searcher.getVisitorParameters( - newQuery("visit?visit.selection=id.user=1234&visit.cluster=unknown"), null); - - assertTrue(false); - } catch (Exception e) { - // e.printStackTrace(); - } - } - - - @Test(expected = IllegalArgumentException.class) - public void testNoClusterParamWhenSeveralClusters() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - ClusterListConfig.Storage.Builder storageCluster1 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foo").name("foo"); - ClusterListConfig.Storage.Builder storageCluster2 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.bar").name("bar"); - ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(Arrays.asList(storageCluster1, storageCluster2))); - ClusterList clusterList = new ClusterList(clusterListCfg); - VisitSearcher searcher = new VisitSearcher(new FeedContext( - new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(100).route("whatever").retryenabled(true)), - new LoadTypeConfig(new LoadTypeConfig.Builder())), - factory, docMan, clusterList, new NullFeedMetric(true))); - - searcher.getVisitorParameters(newQuery("visit?visit.selection=id.user=1234"), null); - } - - @Test - public void testSimple() throws Exception { - Chain<Searcher> searchChain = new Chain<>(create()); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("visit?visit.selection=id.user=1234&hits=100")); - assertEquals(1, result.hits().size()); - assertRendered( - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:foo:1234:bar\"/>\n" + - "</result>\n", result); - } - - private Result invokeVisitRemovesSearchChain() throws Exception { - Chain<Searcher> searchChain = new Chain<>(create()); - return new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("visit?visit.selection=id.user=1234&hits=100&visit.visitRemoves=true")); - } - - @Test - public void visitRemovesIncludesRemoveEntriesInResultXml() throws Exception { - Result result = invokeVisitRemovesSearchChain(); - assertEquals(2, result.hits().size()); - assertRendered( - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:foo:1234:bar\"/>\n" + - "<remove documentid=\"userdoc:foo:1234:removed\"/>\n" + - "</result>\n", result); - } - - @Test - public void removedDocumentIdsAreXmlEscaped() throws Exception { - factory = mock(DocumentSessionFactory.class); - when(factory.createVisitorSession(any(VisitorParameters.class))).thenAnswer((p) -> { - VisitorParameters params = (VisitorParameters)p.getArguments()[0]; - DummyVisitorSession session = new DummyVisitorSession(params, docType); - session.clearAutoReplyMessages(); - session.addRemoveReply("userdoc:foo:1234:<rem\"o\"ved&stuff>"); - return session; - }); - Result result = invokeVisitRemovesSearchChain(); - assertEquals(1, result.hits().size()); - assertRendered( - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<remove documentid=\"userdoc:foo:1234:<rem"o"ved&stuff>\"/>\n" + - "</result>\n", result); - } - - private Result invokeSearcherWithUserQuery() throws Exception { - Chain<Searcher> searchChain = new Chain<>(create()); - return new Execution(searchChain, Execution.Context.createContextStub()) - .search(new Query("visit?visit.selection=id.user=1234&hits=100")); - } - - @Test - public void waitUntilDoneFailureReturnsTimeoutErrorHit() throws Exception { - VisitorSession session = mock(VisitorSession.class); - when(session.waitUntilDone(anyLong())).thenReturn(false); - factory = mock(DocumentSessionFactory.class); - when(factory.createVisitorSession(any(VisitorParameters.class))).thenReturn(session); - - Result result = invokeSearcherWithUserQuery(); - assertNotNull(result.hits().getErrorHit()); - assertEquals(Error.TIMEOUT.code, result.hits().getErrorHit().errors().iterator().next().getCode()); - } - - @Test - @SuppressWarnings("deprecation") - public void testRendererWiring() throws Exception { - Chain<Searcher> searchChain = new Chain<>(create()); - { - Query query = newQuery("visit?visit.selection=id.user=1234&hits=100&format=json"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - assertEquals(com.yahoo.prelude.templates.DefaultTemplateSet.class, result.getTemplating().getTemplates().getClass()); - } - { - Query query = newQuery("visit?visit.selection=id.user=1234&hits=100&format=JsonRenderer"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - assertEquals(com.yahoo.prelude.templates.DefaultTemplateSet.class, result.getTemplating().getTemplates().getClass()); - } - { - Query query = newQuery("visit?visit.selection=id.user=1234&hits=100"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - assertEquals(DocumentXMLTemplate.class, result.getTemplating().getTemplates().getClass()); - } - } - - public static void assertRendered(String expected, Result result) throws Exception { - assertEquals(expected, ResultRenderingUtil.getRendered(result)); - } - - private Query newQuery(String queryString) { - return new Query(HttpRequest.createTestRequest(queryString, com.yahoo.jdisc.http.HttpRequest.Method.GET)); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java deleted file mode 100644 index 47745b29032..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.server; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.messagebus.SessionCache; -import com.yahoo.container.logging.AccessLog; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.jdisc.ReferencedResource; -import com.yahoo.jdisc.References; -import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.messagebus.*; -import com.yahoo.messagebus.Error; -import com.yahoo.messagebus.shared.SharedMessageBus; -import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.text.Utf8; -import com.yahoo.text.Utf8String; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; -import com.yahoo.vespaxmlparser.MockFeedReaderFactory; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.junit.Assert.assertEquals; - -/** - * Check FeedHandler APIs. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class V2ErrorsInResultTestCase { - - LessConfiguredHandler handler; - ExecutorService workers; - - @Before - public void setUp() throws Exception { - workers = Executors.newCachedThreadPool(); - handler = new LessConfiguredHandler(workers); - } - - @After - public void tearDown() throws Exception { - handler.destroy(); - workers.shutdown(); - } - - private static class LessConfiguredHandler extends FeedHandler { - - public LessConfiguredHandler(Executor executor) throws Exception { - super(new FeedHandler.Context(executor, - AccessLog.voidAccessLog(), - new DummyMetric()), - null, null, null, MetricReceiver.nullImplementation); - } - - - @Override - protected Feeder createFeeder(HttpRequest request, InputStream requestInputStream, - BlockingQueue<OperationStatus> operations, String clientId, - boolean sessionIdWasGeneratedJustNow, int protocolVersion) - throws Exception { - return new LessConfiguredFeeder(requestInputStream, operations, - popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, - sourceSessionParams(request), null, this, this.feedReplyHandler, ""); - } - - @Override - protected DocumentTypeManager createDocumentManager( - DocumentmanagerConfig documentManagerConfig) { - return null; - } - } - - private static class MockSharedSession extends SharedSourceSession { - int count; - - public MockSharedSession(SourceSessionParams params) { - super(new SharedMessageBus(new MessageBus(new MockNetwork(), - new MessageBusParams())), params); - count = 0; - } - - @Override - public Result sendMessageBlocking(Message msg) throws InterruptedException { - return sendMessage(msg); - } - - @Override - public Result sendMessage(Message msg) { - Result r; - ReplyHandler handler = msg.popHandler(); - - switch (count++) { - case 0: - r = new Result(ErrorCode.FATAL_ERROR, - "boom"); - break; - case 1: - r = new Result(ErrorCode.TRANSIENT_ERROR, - "transient boom"); - break; - case 2: - final FailedReply reply = new FailedReply(msg.getContext()); - reply.addError(new Error( - ErrorCode.FATAL_ERROR, - "bad mojo, dude")); - handler.handleReply(reply); - r = Result.ACCEPTED; - break; - default: - handler.handleReply(new MockReply(msg.getContext())); - r = Result.ACCEPTED; - } - return r; - } - - } - - private static class FailedReply extends Reply { - Object context; - - public FailedReply(Object context) { - this.context = context; - } - - @Override - public Utf8String getProtocol() { - return null; - } - - @Override - public int getType() { - return 0; - } - - @Override - public Object getContext() { - return context; - } - } - - private static class LessConfiguredFeeder extends Feeder { - - public LessConfiguredFeeder(InputStream stream, - BlockingQueue<OperationStatus> operations, - ClientState storedState, FeederSettings settings, - String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, - SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, - String localHostname) throws Exception { - super(stream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, - sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); - } - - protected ReferencedResource<SharedSourceSession> retainSession( - SourceSessionParams sessionParams, SessionCache sessionCache) { - final SharedSourceSession session = new MockSharedSession(sessionParams); - return new ReferencedResource<>(session, References.fromResource(session)); - } - } - - @Test - public final void test() throws IOException { - String sessionId; - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 ERROR boom \n", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 TRANSIENT_ERROR transient{20}boom \n", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 ERROR bad{20}mojo,{20}dude \n", - Utf8.toString(out.toByteArray())); - } - - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java deleted file mode 100644 index 9960d98f7f1..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java +++ /dev/null @@ -1,530 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.server; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.messagebus.SessionCache; -import com.yahoo.container.logging.AccessLog; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; -import com.yahoo.jdisc.ReferencedResource; -import com.yahoo.jdisc.References; -import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.MessageBus; -import com.yahoo.messagebus.MessageBusParams; -import com.yahoo.messagebus.ReplyHandler; -import com.yahoo.messagebus.Result; -import com.yahoo.messagebus.SourceSessionParams; -import com.yahoo.messagebus.network.Network; -import com.yahoo.messagebus.shared.SharedMessageBus; -import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.config.FeedParams.DataFormat; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; -import com.yahoo.vespaxmlparser.MockFeedReaderFactory; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.logging.Handler; -import java.util.logging.Level; -import java.util.logging.LogRecord; -import java.util.logging.Logger; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; - -/** - * Check FeedHandler APIs. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class V2ExternalFeedTestCase { - - LessConfiguredHandler handler; - ExecutorService workers; - Level logLevel; - Logger logger; - boolean initUseParentHandlers; - LogBuffer logChecker; - - @Before - public void setUp() throws Exception { - workers = Executors.newCachedThreadPool(); - handler = new LessConfiguredHandler(workers); - logger = Logger.getLogger(Feeder.class.getName()); - logLevel = logger.getLevel(); - logger.setLevel(Level.ALL); - initUseParentHandlers = logger.getUseParentHandlers(); - logChecker = new LogBuffer(); - logger.setUseParentHandlers(false); - logger.addHandler(logChecker); - } - - @After - public void tearDown() throws Exception { - handler.destroy(); - workers.shutdown(); - logger.setLevel(logLevel); - logger.removeHandler(logChecker); - logger.setUseParentHandlers(initUseParentHandlers); - } - - private static class LogBuffer extends Handler { - public final BlockingQueue<LogRecord> records = new LinkedBlockingQueue<>(); - - @Override - public void publish(LogRecord record) { - try { - records.put(record); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - @Override - public void flush() { - } - - @Override - public void close() throws SecurityException { - } - } - - private static class LessConfiguredHandler extends FeedHandler { - volatile DataFormat lastFormatSeen; - - public LessConfiguredHandler(Executor executor) throws Exception { - super(new FeedHandler.Context(executor, - AccessLog.voidAccessLog(), - new DummyMetric()), - null, null, null, MetricReceiver.nullImplementation); - } - - @Override - protected Feeder createFeeder(HttpRequest request, - InputStream requestInputStream, - BlockingQueue<OperationStatus> operations, - String clientId, - boolean sessionIdWasGeneratedJustNow, int protocolVersion) - throws Exception { - LessConfiguredFeeder f = new LessConfiguredFeeder(requestInputStream, operations, - popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, - sourceSessionParams(request), null, this, this.feedReplyHandler, "ourHostname"); - lastFormatSeen = f.settings.dataFormat; - return f; - } - - @Override - protected DocumentTypeManager createDocumentManager( - DocumentmanagerConfig documentManagerConfig) { - return null; - } - } - - private static class MockSharedSession extends SharedSourceSession { - - public MockSharedSession(SourceSessionParams params) { - super(new SharedMessageBus(new MessageBus(new MockNetwork(), - new MessageBusParams())), params); - } - - @Override - public Result sendMessageBlocking(Message msg) throws InterruptedException { - return sendMessage(msg); - } - - @Override - public Result sendMessage(Message msg) { - ReplyHandler handler = msg.popHandler(); - MockReply mockReply = new MockReply(msg.getContext()); - if (msg instanceof Feeder.FeedErrorMessage) { - mockReply.addError(new com.yahoo.messagebus.Error(123, "Could not feed this")); - } - if (msg instanceof PutDocumentMessage) { - assert(msg.getTrace().getLevel() == 4); - assert(((PutDocumentMessage) msg).getPriority().name().equals("LOWEST")); - } - handler.handleReply(mockReply); - return Result.ACCEPTED; - } - - } - - private static class LessConfiguredFeeder extends Feeder { - public LessConfiguredFeeder(InputStream stream, - BlockingQueue<OperationStatus> operations, - ClientState storedState, FeederSettings settings, - String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, - SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, - String localHostname) throws Exception { - super(stream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, - sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); - } - - protected ReferencedResource<SharedSourceSession> retainSession( - SourceSessionParams sessionParams, SessionCache sessionCache) { - final SharedSourceSession session = new MockSharedSession(sessionParams); - return new ReferencedResource<>(session, References.fromResource(session)); - } - } - - @Test - public final void test() throws IOException, InterruptedException { - String sessionId; - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[]{1, 3, 2}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n", - Utf8.toString(out.toByteArray())); - assertEquals("text/plain", r.getContentType()); - assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding()); - assertEquals(7, logChecker.records.size()); - String actualHandshake = logChecker.records.take().getMessage(); - assertThat(actualHandshake, actualHandshake.matches("Handshake completed for client (-?)(.+?)-#(.*?)\\."), is(true)); - assertEquals("Successfully deserialized document id: id:banana:banana::doc1", - logChecker.records.take().getMessage()); - assertEquals("Sent message successfully, document id: id:banana:banana::doc1", - logChecker.records.take().getMessage()); - } - - //test session ID without #, i.e. something fishy related to VIPs is going on - sessionId = "something"; - - { - InputStream in = new MetaStream(new byte[]{1, 3, 2}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "2"); - - HttpResponse r = handler.handle(nalle); - r.render(out); - String expectedErrorMsg = "Got request from client with id 'something', but found no session for this client."; - assertThat(Utf8.toString(out.toByteArray()), containsString(expectedErrorMsg)); - assertEquals("text/plain", r.getContentType()); - assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding()); - } - - //test session ID with trailing # but no hostname - sessionId = "something#"; - - { - InputStream in = new MetaStream(new byte[]{1, 3, 2}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); - HttpResponse r = handler.handle(nalle); - r.render(out); - String expectedErrorMsg = "Got request from client with id 'something#', but found no session for this client."; - assertThat(Utf8.toString(out.toByteArray()), containsString(expectedErrorMsg)); - assertEquals("text/plain", r.getContentType()); - assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding()); - } - - //test session ID with trailing # and some unknown hostname at the end - sessionId = "something#thisHostnameDoesNotExistAnywhere"; - - { - InputStream in = new MetaStream(new byte[]{1, 3, 2}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); - HttpResponse r = handler.handle(nalle); - r.render(out); - String expectedErrorMsg = "Got request from client with id 'something#thisHostnameDoesNotExistAnywhere', " + - "but found no session for this client. Session was originally established " + - "towards host thisHostnameDoesNotExistAnywhere, but our hostname is " + - "ourHostname."; - assertThat(Utf8.toString(out.toByteArray()), containsString(expectedErrorMsg)); - assertEquals("text/plain", r.getContentType()); - assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding()); - } - - //test session ID with trailing # and some unknown hostname at the end - sessionId = "something#ourHostname"; - - { - InputStream in = new MetaStream(new byte[]{1, 3, 2}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" + - "id:banana:banana::doc1 OK Document{20}processed. \n" + - "id:banana:banana::doc1 OK Document{20}processed. \n", - Utf8.toString(out.toByteArray())); - assertEquals("text/plain", r.getContentType()); - assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding()); - Thread.sleep(1000); - } - } - - @Test - public final void testFailedReading() throws IOException { - String sessionId; - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 4 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 ERROR Could{20}not{20}feed{20}this \n", - Utf8.toString(out.toByteArray())); - } - } - - @Test - public final void testCleaningDoesNotBlowUp() throws IOException { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - handler.forceRunCleanClients(); - } - - @Test - public final void testMockNetworkDoesNotBlowUp() { - Network n = new MockNetwork(); - n.registerSession(null); - n.unregisterSession(null); - assertTrue(n.allocServiceAddress(null)); - n.freeServiceAddress(null); - n.send(null, null); - assertNull(n.getConnectionSpec()); - assertNull(n.getMirror()); - } - - @Test - public final void testMockReplyDoesNotBlowUp() { - MockReply r = new MockReply(null); - assertNull(r.getProtocol()); - assertEquals(0, r.getType()); - assertFalse(r.hasFatalErrors()); - } - - @Test - public final void testFlush() throws IOException { - String sessionId; - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 1, 1, 1, 1, 1, 1, 1}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n", - Utf8.toString(out.toByteArray())); - } - } - - @Test - public final void testIllegalVersion() throws IOException { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers() - .add(Headers.VERSION, Integer.toString(Integer.MAX_VALUE)); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals(Headers.HTTP_NOT_ACCEPTABLE, r.getStatus()); - } - - @Test - public final void testSettings() { - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - nalle.getJDiscRequest().headers().add(Headers.ROUTE, "bamse brakar"); - nalle.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true"); - FeederSettings settings = new FeederSettings(nalle); - assertEquals(false, settings.drain); - assertEquals(2, settings.route.getNumHops()); - assertEquals(true, settings.denyIfBusy); - } - - @Test - public final void testJsonInputFormat() throws IOException, InterruptedException { - String sessionId; - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[]{1, 3, 2}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DATA_FORMAT, DataFormat.JSON_UTF8.name()); - nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n", - Utf8.toString(out.toByteArray())); - assertEquals("text/plain", r.getContentType()); - assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding()); - assertEquals(7, logChecker.records.size()); - String actualHandshake = logChecker.records.take().getMessage(); - assertThat(actualHandshake, actualHandshake.matches("Handshake completed for client (-?)(.+?)-#(.*?)\\."), is(true)); - assertEquals("Successfully deserialized document id: id:banana:banana::doc1", - logChecker.records.take().getMessage()); - assertEquals("Sent message successfully, document id: id:banana:banana::doc1", - logChecker.records.take().getMessage()); - assertSame(DataFormat.JSON_UTF8, handler.lastFormatSeen); - } - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java deleted file mode 100644 index 6290c22f694..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.server; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.messagebus.SessionCache; -import com.yahoo.container.logging.AccessLog; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.jdisc.ReferencedResource; -import com.yahoo.jdisc.References; -import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.messagebus.*; -import com.yahoo.messagebus.shared.SharedMessageBus; -import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; -import com.yahoo.vespaxmlparser.MockFeedReaderFactory; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.junit.Assert.assertEquals; - -/** - * Check FeedHandler APIs. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class V2FailingMessagebusTestCase { - - LessConfiguredHandler handler; - ExecutorService workers; - int mbus; - - @Before - public void setUp() throws Exception { - workers = Executors.newCachedThreadPool(); - handler = new LessConfiguredHandler(workers); - mbus = 0; - } - - @After - public void tearDown() throws Exception { - handler.destroy(); - workers.shutdown(); - mbus = 0; - } - - private class LessConfiguredHandler extends FeedHandler { - - public LessConfiguredHandler(Executor executor) throws Exception { - super(new FeedHandler.Context(executor, AccessLog.voidAccessLog(), new DummyMetric()), - null, null, null, MetricReceiver.nullImplementation); - } - - @Override - protected Feeder createFeeder(HttpRequest request, - InputStream requestInputStream, - BlockingQueue<OperationStatus> operations, - String clientId, - boolean sessionIdWasGeneratedJustNow, int protocolVersion) throws Exception { - return new LessConfiguredFeeder(requestInputStream, operations, - popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, - sourceSessionParams(request), null, this, this.feedReplyHandler, ""); - } - - @Override - protected DocumentTypeManager createDocumentManager( - DocumentmanagerConfig documentManagerConfig) { - return null; - } - } - - private class MockSharedSession extends SharedSourceSession { - - public MockSharedSession(SourceSessionParams params) { - super(new SharedMessageBus(new MessageBus(new MockNetwork(), - new MessageBusParams())), params); - } - - @Override - public Result sendMessageBlocking(Message msg) throws InterruptedException { - return sendMessage(msg); - } - - @Override - public Result sendMessage(Message msg) { - ReplyHandler handler = msg.popHandler(); - - switch (mbus) { - case 0: - throw new RuntimeException("boom"); - case 1: - Result r = new Result(ErrorCode.SEND_QUEUE_FULL, "tralala"); - mbus = 2; - return r; - case 2: - handler.handleReply(new MockReply(msg.getContext())); - return Result.ACCEPTED; - default: - throw new IllegalStateException("WTF?!"); - } - } - } - - private class LessConfiguredFeeder extends Feeder { - - public LessConfiguredFeeder(InputStream inputStream, - BlockingQueue<OperationStatus> operations, - ClientState storedState, FeederSettings settings, - String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, - SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, - String localHostname) throws Exception { - super(inputStream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, - sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); - } - - protected ReferencedResource<SharedSourceSession> retainSession( - SourceSessionParams sessionParams, SessionCache sessionCache) { - final SharedSourceSession session = new MockSharedSession(sessionParams); - return new ReferencedResource<>(session, References.fromResource(session)); - } - } - - @Test - public final void testFailingMbus() throws IOException { - String sessionId; - { - InputStream in = new MetaStream(new byte[]{1}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[]{1}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 ERROR boom \n", - Utf8.toString(out.toByteArray())); - } - } - - @Test - public final void testBusyMbus() throws IOException { - String sessionId; - { - InputStream in = new MetaStream(new byte[]{1}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - mbus = 2; - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - mbus = 1; - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - nalle.getJDiscRequest().headers() - .add(Headers.DENY_IF_BUSY, "false"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - mbus = 1; - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - nalle.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 TRANSIENT_ERROR tralala \n", - Utf8.toString(out.toByteArray())); - } - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java deleted file mode 100644 index 633477dcc79..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.server; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.messagebus.SessionCache; -import com.yahoo.container.logging.AccessLog; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.jdisc.ReferencedResource; -import com.yahoo.jdisc.References; -import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.messagebus.*; -import com.yahoo.messagebus.Error; -import com.yahoo.messagebus.shared.SharedMessageBus; -import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.junit.Assert.assertEquals; - -/** - * Check FeedHandler APIs. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class V2NoXmlReaderTestCase { - - LessConfiguredHandler handler; - ExecutorService workers; - - @Before - public void setUp() throws Exception { - workers = Executors.newCachedThreadPool(); - handler = new LessConfiguredHandler(workers); - } - - @After - public void tearDown() throws Exception { - handler.destroy(); - workers.shutdown(); - } - - private static class LessConfiguredHandler extends FeedHandler { - - public LessConfiguredHandler(Executor executor) throws Exception { - super(new FeedHandler.Context(executor, AccessLog.voidAccessLog(), new DummyMetric()), - null, null, null, MetricReceiver.nullImplementation); - } - - - @Override - protected Feeder createFeeder(HttpRequest request, InputStream requestInputStream, - BlockingQueue<OperationStatus> operations, String clientId, - boolean sessionIdWasGeneratedJustNow, int protocolVersion) - throws Exception { - return new LessConfiguredFeeder(requestInputStream, operations, - popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, - sourceSessionParams(request), null, this, this.feedReplyHandler, ""); - } - - @Override - protected DocumentTypeManager createDocumentManager( - DocumentmanagerConfig documentManagerConfig) { - return null; - } - } - - private static class MockSharedSession extends SharedSourceSession { - - public MockSharedSession(SourceSessionParams params) { - super(new SharedMessageBus(new MessageBus(new MockNetwork(), - new MessageBusParams())), params); - } - - @Override - public Result sendMessageBlocking(Message msg) throws InterruptedException { - return sendMessage(msg); - } - - @Override - public Result sendMessage(Message msg) { - ReplyHandler handler = msg.popHandler(); - MockReply mockReply = new MockReply(msg.getContext()); - if (msg instanceof Feeder.FeedErrorMessage) { - mockReply.addError(new Error(123, "Could not feed this")); - } - handler.handleReply(mockReply); - return Result.ACCEPTED; - } - - } - - private static class LessConfiguredFeeder extends Feeder { - - public LessConfiguredFeeder(InputStream inputStream, - BlockingQueue<OperationStatus> operations, - ClientState storedState, FeederSettings settings, - String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, - SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, - String localHostname) throws Exception { - super(inputStream, null, null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, - sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); - } - - protected ReferencedResource<SharedSourceSession> retainSession( - SourceSessionParams sessionParams, SessionCache sessionCache) { - final SharedSourceSession session = new MockSharedSession(sessionParams); - return new ReferencedResource<>(session, References.fromResource(session)); - } - } - - @Test - public final void test() throws IOException { - String sessionId; - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - //This is different from v1. If we cannot parse XML, we will still get response code 200, but with a sensible - //error message in the response. - assertEquals(200, r.getStatus()); - assertEquals("id:banana:banana::doc1 ERROR Could{20}not{20}feed{20}this \n", - Utf8.toString(out.toByteArray())); - } - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java index d6f605b0379..128664dda9e 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java @@ -23,13 +23,14 @@ public class VersionsTestCase { private static final List<String> EMPTY = Collections.emptyList(); private static final List<String> ONE_TWO = Arrays.asList("1", "2"); + private static final List<String> ONE_THREE = Arrays.asList("1", "3"); private static final List<String> TWO_THREE = Arrays.asList("3", "2"); - private static final List<String> ONE_NULL_TWO = Arrays.asList("1", null, "2"); - private static final List<String> ONE_COMMA_TWO = Collections.singletonList("1, 2"); - private static final List<String> ONE_EMPTY_TWO = Arrays.asList("1", "", "2"); + private static final List<String> ONE_NULL_THREE = Arrays.asList("1", null, "3"); + private static final List<String> ONE_COMMA_THREE = Collections.singletonList("1, 3"); + private static final List<String> ONE_EMPTY_THREE = Arrays.asList("1", "", "3"); private static final List<String> TOO_LARGE_NUMBER = Collections.singletonList("1000000000"); - private static final List<String> TWO_TOO_LARGE_NUMBER = Arrays.asList("2", "1000000000"); - private static final List<String> TWO_COMMA_TOO_LARGE_NUMBER = Arrays.asList("2,1000000000"); + private static final List<String> THREE_TOO_LARGE_NUMBER = Arrays.asList("3", "1000000000"); + private static final List<String> THREE_COMMA_TOO_LARGE_NUMBER = Arrays.asList("3,1000000000"); private static final List<String> GARBAGE = Collections.singletonList("garbage"); @Test @@ -42,8 +43,15 @@ public class VersionsTestCase { @Test public void testOneTwo() throws Exception { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_TWO); + assertThat(v.first, instanceOf(ErrorHttpResponse.class)); + assertThat(v.second, is(-1)); + } + + @Test + public void testOneThree() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_THREE); assertThat(v.first, nullValue()); - assertThat(v.second, is(2)); + assertThat(v.second, is(3)); } @Test @@ -54,24 +62,24 @@ public class VersionsTestCase { } @Test - public void testOneNullTwo() throws Exception { - Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_NULL_TWO); + public void testOneNullThree() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_NULL_THREE); assertThat(v.first, nullValue()); - assertThat(v.second, is(2)); + assertThat(v.second, is(3)); } @Test - public void testOneCommaTwo() throws Exception { - Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_COMMA_TWO); + public void testOneCommaThree() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_COMMA_THREE); assertThat(v.first, nullValue()); - assertThat(v.second, is(2)); + assertThat(v.second, is(3)); } @Test - public void testOneEmptyTwo() throws Exception { - Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_EMPTY_TWO); + public void testOneEmptyThree() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_EMPTY_THREE); assertThat(v.first, nullValue()); - assertThat(v.second, is(2)); + assertThat(v.second, is(3)); } @Test @@ -83,22 +91,22 @@ public class VersionsTestCase { errorResponse.render(errorMsg); assertThat(errorMsg.toString(), is("Could not parse X-Yahoo-Feed-Protocol-Versionheader of request (values: [1000000000]). " + - "Server supports protocol versions [2, 3]")); + "Server supports protocol versions [3]")); assertThat(v.second, is(-1)); } @Test - public void testTwoTooLarge() throws Exception { - Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_TOO_LARGE_NUMBER); + public void testThreeTooLarge() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(THREE_TOO_LARGE_NUMBER); assertThat(v.first, nullValue()); - assertThat(v.second, is(2)); + assertThat(v.second, is(3)); } @Test public void testTwoCommaTooLarge() throws Exception { - Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_COMMA_TOO_LARGE_NUMBER); + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(THREE_COMMA_TOO_LARGE_NUMBER); assertThat(v.first, nullValue()); - assertThat(v.second, is(2)); + assertThat(v.second, is(3)); } @Test |