summaryrefslogtreecommitdiffstats
path: root/vespaclient-container-plugin
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@oath.com>2018-11-08 16:42:45 +0100
committergjoranv <gv@oath.com>2019-01-21 15:09:25 +0100
commitbb26b9f9a5ba9175b6b81a0c8c2f54de19002cc8 (patch)
tree673c7d2bff372abfd3346e15477fc202bb181614 /vespaclient-container-plugin
parent6c5f6006b7684ff22bc21b7e66894a734af3b922 (diff)
Remove old feedapi
Diffstat (limited to 'vespaclient-container-plugin')
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java65
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java41
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java34
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java91
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java91
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java89
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java36
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/package-info.java3
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java31
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java98
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java45
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java23
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java120
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java508
-rwxr-xr-xvespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java39
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java210
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java6
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java1
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore0
-rwxr-xr-xvespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java1064
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java104
-rwxr-xr-xvespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java129
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java98
-rwxr-xr-xvespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java1093
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java23
-rw-r--r--vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java243
26 files changed, 1 insertions, 4284 deletions
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java
deleted file mode 100755
index 365f605688c..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.feedhandler;
-
-import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.metrics.MetricManager;
-import com.yahoo.metrics.MetricSnapshot;
-import com.yahoo.text.Utf8String;
-import com.yahoo.text.XMLWriter;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-
-/**
- * @deprecated Legacy API. Will be removed in Vespa 7
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class StatusResponse extends HttpResponse {
-
- MetricManager manager;
- int verbosity;
- int snapshotTime;
-
- StatusResponse(MetricManager manager, int verbosity, int snapshotTime) {
- super(com.yahoo.jdisc.http.HttpResponse.Status.OK);
- this.manager = manager;
- this.snapshotTime = snapshotTime;
- this.verbosity = verbosity;
- }
-
- @Override
- public void render(OutputStream stream) throws IOException {
- XMLWriter writer = new XMLWriter(new OutputStreamWriter(stream));
- writer.openTag("status");
- if (verbosity >= 2) {
- writer.attribute(new Utf8String("description"), "Metrics since start");
- }
-
- if (snapshotTime == 0) {
- MetricSnapshot snapshot = (new MetricSnapshot(
- "Total metrics from start until current time", 0,
- manager.getActiveMetrics().getMetrics(), false));
- manager.getTotalMetricSnapshot().addToSnapshot(snapshot, (int)(System.currentTimeMillis() / 1000), false);
- snapshot.printXml(manager, "", verbosity, writer);
- } else {
- try {
- manager.getMetricSnapshotSet(snapshotTime).getSnapshot().printXml(manager, "", verbosity, writer);
- } catch (Exception e) {
- writer.openTag("error");
- writer.attribute(new Utf8String("details"), "No metric snapshot with period " + snapshotTime +
- " was found. Legal snapshot periods are: " + manager.getSnapshotPeriods());
- writer.closeTag();
- }
- }
- writer.closeTag();
- writer.flush();
- }
-
- @Override
- public java.lang.String getContentType() {
- return "application/xml";
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java
deleted file mode 100755
index 8fe721899f9..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java
+++ /dev/null
@@ -1,41 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.feedhandler;
-
-import java.util.concurrent.Executor;
-import javax.inject.Inject;
-
-import com.yahoo.jdisc.Metric;
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
-
-/**
- * @deprecated Legacy API. Will be removed in Vespa 7
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class VespaFeedHandlerCompatibility extends ThreadedHttpRequestHandler {
-
- private final VespaFeedHandlerGet getHandler;
- private final VespaFeedHandler feedHandler;
-
- @Inject
- public VespaFeedHandlerCompatibility(Executor executor, Metric metric, VespaFeedHandlerGet getHandler,
- VespaFeedHandler feedHandler) {
- super(executor, metric);
- this.getHandler = getHandler;
- this.feedHandler = feedHandler;
- }
-
- @Override
- public HttpResponse handle(HttpRequest request) {
- boolean hasType = request.hasProperty("type");
- // If we have an ID and no document type, redirect to Get
- if (request.hasProperty("id") && !hasType) {
- return getHandler.handle(request);
- } else {
- return feedHandler.handle(request);
- }
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java
deleted file mode 100755
index ab8728e017b..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.feedhandler;
-
-import java.util.Collections;
-import java.util.concurrent.Executor;
-import javax.inject.Inject;
-
-import com.yahoo.jdisc.Metric;
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
-import com.yahoo.search.handler.SearchHandler;
-
-/**
- * @deprecated Legacy API. Will be removed in Vespa 7
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class VespaFeedHandlerGet extends ThreadedHttpRequestHandler {
-
- private final SearchHandler searchHandler;
-
- @Inject
- public VespaFeedHandlerGet(SearchHandler searchHandler, Executor executor, Metric metric) {
- super(executor, metric, true);
- this.searchHandler = searchHandler;
- }
-
- @Override
- public HttpResponse handle(HttpRequest request) {
- return searchHandler.handle(new HttpRequest(request.getJDiscRequest(), request.getData(), Collections.singletonMap("searchChain", "vespaget")));
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java
deleted file mode 100755
index 9baad1a605d..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java
+++ /dev/null
@@ -1,91 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.feedhandler;
-
-import com.google.inject.Inject;
-import com.yahoo.clientmetrics.RouteMetricSet;
-import com.yahoo.cloud.config.ClusterListConfig;
-import com.yahoo.cloud.config.SlobroksConfig;
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.container.protect.Error;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.config.DocumentmanagerConfig;
-import com.yahoo.feedapi.FeedContext;
-import com.yahoo.feedapi.MessagePropertyProcessor;
-import com.yahoo.feedapi.SingleSender;
-import com.yahoo.jdisc.Metric;
-import com.yahoo.vespa.config.content.LoadTypeConfig;
-import com.yahoo.vespaclient.config.FeederConfig;
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.util.concurrent.Executor;
-
-/**
- * @deprecated Legacy API. Will be removed in Vespa 7
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class VespaFeedHandlerRemove extends VespaFeedHandlerBase {
-
- @Inject
- public VespaFeedHandlerRemove(FeederConfig feederConfig,
- LoadTypeConfig loadTypeConfig,
- DocumentmanagerConfig documentmanagerConfig,
- SlobroksConfig slobroksConfig,
- ClusterListConfig clusterListConfig,
- Executor executor,
- Metric metric) throws Exception {
- super(feederConfig, loadTypeConfig, documentmanagerConfig, slobroksConfig, clusterListConfig, executor, metric);
- }
-
- VespaFeedHandlerRemove(FeedContext context, Executor executor) throws Exception {
- super(context, executor);
- }
-
- @Override
- public HttpResponse handle(HttpRequest request) {
- if (request.getProperty("status") != null) {
- return new MetricResponse(context.getMetrics().getMetricSet());
- }
-
- MessagePropertyProcessor.PropertySetter properties = getPropertyProcessor().buildPropertySetter(request);
- String route = properties.getRoute().toString();
- FeedResponse response = new FeedResponse(new RouteMetricSet(route, null));
- SingleSender sender = new SingleSender(response, getSharedSender(route));
- sender.addMessageProcessor(properties);
-
- response.setAbortOnFeedError(properties.getAbortOnFeedError());
-
- if (request.hasProperty("id")) {
- sender.remove(new DocumentId(request.getProperty("id")));
- } else if (request.hasProperty("id[0]")) {
- int index = 0;
- while (request.hasProperty("id[" + index + "]")) {
- sender.remove(new DocumentId(request.getProperty("id[" + index + "]")));
- ++index;
- }
- }
-
- if (request.getData() != null) {
- try {
- String line;
- BufferedReader reader = new BufferedReader(
- new InputStreamReader(getRequestInputStream(request), "UTF-8"));
- while ((line = reader.readLine()) != null) {
- sender.remove(new DocumentId(line));
- }
- } catch (Exception e) {
- response.addError(e.getClass() + ": " + e.getCause());
- }
- }
-
- sender.done();
- long millis = getTimeoutMillis(request);
- boolean completed = sender.waitForPending(millis);
- if ( ! completed)
- response.addError(Error.TIMEOUT, "Timed out after "+millis+" ms waiting for responses");
- return response;
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java
deleted file mode 100644
index 0ca77decf22..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java
+++ /dev/null
@@ -1,91 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.feedhandler;
-
-import com.google.inject.Inject;
-import com.yahoo.clientmetrics.RouteMetricSet;
-import com.yahoo.cloud.config.ClusterListConfig;
-import com.yahoo.cloud.config.SlobroksConfig;
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.container.protect.Error;
-import com.yahoo.document.config.DocumentmanagerConfig;
-import com.yahoo.documentapi.messagebus.protocol.RemoveLocationMessage;
-import com.yahoo.feedapi.FeedContext;
-import com.yahoo.feedapi.MessagePropertyProcessor;
-import com.yahoo.feedapi.SingleSender;
-import com.yahoo.jdisc.Metric;
-import com.yahoo.messagebus.routing.Route;
-import com.yahoo.vespa.config.content.LoadTypeConfig;
-import com.yahoo.vespaclient.config.FeederConfig;
-
-import java.util.concurrent.Executor;
-
-/**
- * @deprecated Legacy API. Will be removed in Vespa 7
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class VespaFeedHandlerRemoveLocation extends VespaFeedHandlerBase {
-
- @Inject
- public VespaFeedHandlerRemoveLocation(FeederConfig feederConfig,
- LoadTypeConfig loadTypeConfig,
- DocumentmanagerConfig documentmanagerConfig,
- SlobroksConfig slobroksConfig,
- ClusterListConfig clusterListConfig,
- Executor executor, Metric metric) throws Exception {
- super(feederConfig, loadTypeConfig, documentmanagerConfig, slobroksConfig, clusterListConfig, executor, metric);
- }
-
- VespaFeedHandlerRemoveLocation(FeedContext context, Executor executor) throws Exception {
- super(context, executor);
- }
-
- @Override
- public HttpResponse handle(HttpRequest request) {
- MessagePropertyProcessor.PropertySetter properties = getPropertyProcessor().buildPropertySetter(request);
- FeedResponse response;
-
- if (request.getProperty("route") == null) {
- if (context.getClusterList().getStorageClusters().size() == 0) {
- return new FeedResponse(null).addError("No storage clusters configured and no alternate route specified.");
- } else if (context.getClusterList().getStorageClusters().size() > 1) {
- return new FeedResponse(null).addError("More than one storage cluster configured and no route specified.");
- } else {
- properties.setRoute(Route.parse(context.getClusterList().getStorageClusters().get(0).getName()));
- }
- }
-
- response = new FeedResponse(new RouteMetricSet(properties.getRoute().toString(), null));
-
- SingleSender sender = new SingleSender(response, getSharedSender(properties.getRoute().toString()));
- sender.addMessageProcessor(properties);
-
- String user = request.getProperty("user");
- String group = request.getProperty("group");
- String selection = request.getProperty("selection");
-
- boolean oneFound = (user != null) ^ (group != null) ^ (selection != null);
-
- if (!oneFound) {
- response.addError("Exactly one of \"user\", \"group\" or \"selection\" must be specified for removelocation");
- return response;
- }
-
- if (user != null) {
- selection = "id.user=" + user;
- }
- if (group != null) {
- selection = "id.group=\"" + group + "\"";
- }
-
- sender.send(new RemoveLocationMessage(selection));
- sender.done();
- long millis = getTimeoutMillis(request);
- boolean completed = sender.waitForPending(millis);
- if ( ! completed)
- response.addError(Error.TIMEOUT, "Timed out after "+millis+" ms waiting for responses");
- return response;
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java
deleted file mode 100755
index ac16159a9bb..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java
+++ /dev/null
@@ -1,89 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.feedhandler;
-
-import java.util.concurrent.Executor;
-
-import com.yahoo.cloud.config.ClusterListConfig;
-import com.yahoo.cloud.config.SlobroksConfig;
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
-import com.yahoo.document.config.DocumentmanagerConfig;
-import com.yahoo.jdisc.Metric;
-import com.yahoo.vespa.config.content.LoadTypeConfig;
-import com.yahoo.feedapi.FeedContext;
-import com.yahoo.metrics.MetricManager;
-import com.yahoo.metrics.MetricSet;
-import com.yahoo.vespaclient.config.FeederConfig;
-
-/**
- * @deprecated Legacy API. Will be removed in Vespa 7
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class VespaFeedHandlerStatus extends ThreadedHttpRequestHandler {
-
- private MetricManager manager;
-
- public VespaFeedHandlerStatus(FeederConfig feederConfig,
- LoadTypeConfig loadTypeConfig,
- DocumentmanagerConfig documentmanagerConfig,
- SlobroksConfig slobroksConfig,
- ClusterListConfig clusterListConfig,
- Executor executor,
- Metric metric) {
- this(FeedContext.getInstance(feederConfig, loadTypeConfig,
- documentmanagerConfig, slobroksConfig,
- clusterListConfig, metric),
- true, true, executor);
- }
-
- VespaFeedHandlerStatus(FeedContext context, boolean doLog, boolean makeSnapshots, Executor executor) {
- super(executor);
- manager = new MetricManager();
- final MetricSet metricSet = context.getMetrics().getMetricSet();
- metricSet.unregister();
- manager.registerMetric(metricSet);
- if (doLog) {
- manager.addMetricToConsumer("log", "routes.total.putdocument.count");
- manager.addMetricToConsumer("log", "routes.total.removedocument.count");
- manager.addMetricToConsumer("log", "routes.total.updatedocument.count");
- manager.addMetricToConsumer("log", "routes.total.getdocument.count");
-
- manager.addMetricToConsumer("log", "routes.total.putdocument.errors.total");
- manager.addMetricToConsumer("log", "routes.total.removedocument.errors.total");
- manager.addMetricToConsumer("log", "routes.total.updatedocument.errors.total");
- manager.addMetricToConsumer("log", "routes.total.getdocument.errors.total");
-
- manager.addMetricToConsumer("log", "routes.total.putdocument.latency");
- manager.addMetricToConsumer("log", "routes.total.removedocument.latency");
- manager.addMetricToConsumer("log", "routes.total.updatedocument.latency");
- manager.addMetricToConsumer("log", "routes.total.getdocument.latency");
- }
-
- if (doLog || makeSnapshots) {
- new Thread(manager).start();
- }
- }
-
- @Override
- public HttpResponse handle(HttpRequest request) {
- try {
- return new StatusResponse(manager, asInt(request.getProperty("verbosity"), 0), asInt(request.getProperty("snapshotperiod"), 0));
- } catch (Exception e) {
- e.printStackTrace();
- return null;
- }
- }
-
- private int asInt(String value, int defaultValue) {
- if (value == null) return defaultValue;
- return Integer.parseInt(value);
- }
-
- @Override
- public void destroy() {
- manager.stop();
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java
deleted file mode 100644
index b73ea455c32..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.feedhandler;
-
-import java.util.Collections;
-import java.util.concurrent.Executor;
-import javax.inject.Inject;
-
-import com.yahoo.jdisc.Metric;
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
-import com.yahoo.search.handler.SearchHandler;
-
-/**
- * @author thomasg
- *
- * @deprecated Legacy API. Will be removed in Vespa 7
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class VespaFeedHandlerVisit extends ThreadedHttpRequestHandler {
-
- private final SearchHandler searchHandler;
-
- @Inject
- public VespaFeedHandlerVisit(SearchHandler searchHandler, Executor executor, Metric metric) {
- super(executor, metric, true);
- this.searchHandler = searchHandler;
- }
-
- @Override
- public HttpResponse handle(HttpRequest request) {
- return searchHandler.handle(new HttpRequest(request.getJDiscRequest(), request.getData(), Collections.singletonMap("searchChain", "vespavisit")));
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/package-info.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/package-info.java
deleted file mode 100644
index 857e7127b6f..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/package-info.java
+++ /dev/null
@@ -1,3 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-// TODO: This implements the old, deprecated document http API. Remove this package on Vespa 7.
-package com.yahoo.feedhandler;
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java
deleted file mode 100755
index c66af4f02ba..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.storage.searcher;
-
-import com.yahoo.documentapi.ProgressToken;
-import com.yahoo.search.result.Hit;
-import java.io.IOException;
-import java.util.Base64;
-
-/**
- * @deprecated
- */
-@Deprecated // OK
-// TODO: Remove on Vespa 7
-public class ContinuationHit extends Hit {
-
- private final String value;
-
- public ContinuationHit(ProgressToken token) {
- super("continuation");
- value = token.serializeToString();
- }
-
- public static ProgressToken getToken(String continuation) {
- return ProgressToken.fromSerializedString(continuation);
- }
-
- public String getValue() {
- return value;
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java
deleted file mode 100755
index cf35c789f65..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.storage.searcher;
-
-import com.yahoo.document.DataType;
-import com.yahoo.document.Document;
-import com.yahoo.document.Field;
-import com.yahoo.document.datatypes.FieldValue;
-import com.yahoo.document.datatypes.Raw;
-import com.yahoo.io.ByteWriter;
-import com.yahoo.text.XML;
-
-import java.io.IOException;
-import java.io.Writer;
-
-/**
- * Template used to render a single field for a single Document. Fields
- * that are either of type CONTENT or RAW are written directly, while
- * all other fields are wrapped in Vespa XML and escaped.
- *
- * @deprecated use a renderer instead
- */
-@Deprecated // OK
-// TODO: Remove on Vespa 7
-@SuppressWarnings("deprecation")
-public class DocumentFieldTemplate extends com.yahoo.prelude.templates.UserTemplate<Writer> {
-
- Field field;
- String contentType;
- String encoding;
- boolean wrapXml;
-
- public DocumentFieldTemplate(Field field, String contentType, String encoding, boolean wrapXml) {
- super("documentfield", contentType, encoding);
- this.field = field;
- this.contentType = contentType;
- this.encoding = encoding;
- this.wrapXml = wrapXml;
- }
-
- @Override
- public void error(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException {
- // Error shouldn't be handled by this template, but rather
- // delegated to the searcher
- }
-
- @Override
- public Writer wrapWriter(Writer writer) {
- /* TODO: uncomment
- if (!(writer instanceof ByteWriter)) {
- throw new IllegalArgumentException("ByteWriter required, but got " + writer.getClass().getName());
- }
- */
-
- return writer;
- }
-
- @Override
- public void header(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException {
- if (wrapXml) {
- // XML wrapping should only be used for default field rendering
- writer.write("<?xml version=\"1.0\" encoding=\"" + encoding + "\"?>\n");
- writer.write("<result>");
- }
- }
-
- @Override
- public void footer(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException {
- if (wrapXml) {
- writer.write("</result>\n");
- }
- }
-
- @Override
- public void hit(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException {
- DocumentHit hit = (DocumentHit)context.get("hit");
- Document doc = hit.getDocument();
- // Assume field existence has been checked before we ever get here.
- // Also assume that relevant encoding/content type is set
- // appropriately according to the request and the field's content
- // type, as this is immutable in the template set.
- FieldValue value = doc.getFieldValue(field);
- if (field.getDataType() == DataType.RAW) {
- ByteWriter bw = (ByteWriter)writer;
- bw.append(((Raw) value).getByteBuffer().array());
- } else {
- writer.write(XML.xmlEscape(value.toString(), false));
- }
- }
-
- @Override
- public void hitFooter(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException {
- }
-
- @Override
- public void noHits(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException {
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java
deleted file mode 100755
index 514cf055e84..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.storage.searcher;
-
-import com.yahoo.document.Document;
-import com.yahoo.document.Field;
-import com.yahoo.document.datatypes.FieldValue;
-import com.yahoo.search.result.Hit;
-
-import java.util.Iterator;
-import java.util.Map;
-
-@Deprecated // OK
-// TODO: Remove on Vespa 7
-public class DocumentHit extends Hit {
-
- private Document document;
- private int index;
-
- public DocumentHit(Document document, int index) {
- super(document.getId().toString());
- this.document = document;
- this.index = index;
- }
-
- public void populateHitFields() {
- // Create hit fields for all document fields
- Iterator<Map.Entry<Field, FieldValue>> fieldIter = document.iterator();
- while (fieldIter.hasNext()) {
- Map.Entry<Field, FieldValue> field = fieldIter.next();
- setField(field.getKey().getName(), field.getValue());
- }
-
- // Assign an explicit document id field
- setField("documentid", document.getId().toString());
- }
-
- public Document getDocument() {
- return document;
- }
-
- public int getIndex() {
- return index;
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java
deleted file mode 100644
index 73363e68e2b..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.storage.searcher;
-
-import com.yahoo.document.DocumentId;
-import com.yahoo.search.result.Hit;
-
-@Deprecated // OK
-// TODO: Remove on Vespa 7
-public class DocumentRemoveHit extends Hit {
-
- private final DocumentId idOfRemovedDoc;
-
- public DocumentRemoveHit(DocumentId idOfRemovedDoc) {
- super(idOfRemovedDoc.toString());
- this.idOfRemovedDoc = idOfRemovedDoc;
- setField("documentid", idOfRemovedDoc.toString());
- }
-
- public DocumentId getIdOfRemovedDoc() {
- return idOfRemovedDoc;
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java
deleted file mode 100755
index 2366e7059bd..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java
+++ /dev/null
@@ -1,120 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.storage.searcher;
-
-import com.yahoo.log.LogLevel;
-import com.yahoo.search.Result;
-import com.yahoo.search.result.ErrorHit;
-import com.yahoo.search.result.ErrorMessage;
-import com.yahoo.search.result.HitGroup;
-import com.yahoo.search.result.Hit;
-import com.yahoo.text.XML;
-
-import java.io.IOException;
-import java.io.Writer;
-import java.util.logging.Logger;
-
-/**
- * @deprecated use a renderer instead
- */
-@Deprecated // OK
-// TODO: Remove on Vespa 7
-@SuppressWarnings("deprecation")
-public class DocumentXMLTemplate extends com.yahoo.prelude.templates.UserTemplate<Writer> {
-
- private static final Logger log = Logger.getLogger(DocumentXMLTemplate.class.getName());
-
- public DocumentXMLTemplate() {
- super("vespa_xml");
- }
-
- public DocumentXMLTemplate(String mimeType, String encoding) {
- super("vespa_xml", mimeType, encoding);
- }
-
- private void writeErrorMessage(Writer writer, String type, int code,
- String message, String detailedMessage) throws IOException {
- writer.write("<error type=\"" + type + "\" code=\"" + code + "\" message=\"");
- writer.write(XML.xmlEscape(message, true));
- if (detailedMessage != null) {
- writer.write(": ");
- writer.write(XML.xmlEscape(detailedMessage, true));
- }
- writer.write("\"/>\n");
- }
-
- private void writeGenericErrorMessage(Writer writer, ErrorMessage message) throws IOException {
- // A bit dirty, but we don't have to support many different types
- if (message instanceof MessageBusErrorMessage) {
- writeErrorMessage(writer, "messagebus",
- ((MessageBusErrorMessage)message).getMessageBusCode(),
- message.getMessage(), message.getDetailedMessage());
- } else {
- writeErrorMessage(writer, "searcher", message.getCode(),
- message.getMessage(), message.getDetailedMessage());
- }
- }
-
- @Override
- public void error(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException {
- writer.write("<errors>\n");
- // If the error contains no error hits, use a single error with the main
- // code and description. Otherwise, use the error hits explicitly
- ErrorHit errorHit = ((Result)context.get("result")).hits().getErrorHit();
- if (errorHit == null || errorHit.errors().isEmpty()) {
- ErrorMessage message = ((Result)context.get("result")).hits().getError();
- writeGenericErrorMessage(writer, message);
- } else {
- for (ErrorMessage message : errorHit.errors()) {
- writeGenericErrorMessage(writer, message);
- }
- }
- writer.write("</errors>\n");
- }
-
- @Override
- public void header(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException {
- writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
- writer.write("<result>\n");
- HitGroup rootGroup = ((Result) context.get("result")).hits();
- if (rootGroup.getField(VisitSearcher.VISITOR_CONTINUATION_TOKEN_FIELDNAME) != null) {
- writer.write("<continuation>" + rootGroup.getField(VisitSearcher.VISITOR_CONTINUATION_TOKEN_FIELDNAME) + "</continuation>");
- }
- }
-
- @Override
- public void footer(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException {
- writer.write("</result>\n");
- }
-
- @Override
- public void hit(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException {
- Hit hit = (Hit)context.get("hit");
- if (hit instanceof DocumentHit) {
- DocumentHit docHit = (DocumentHit) hit;
- if (docHit.getDocument() != null) {
- writer.write(docHit.getDocument().toXML(" "));
- }
- } else if (hit instanceof DocumentRemoveHit) {
- writeDocumentRemoveHit(writer, (DocumentRemoveHit) hit);
- } else {
- log.log(LogLevel.WARNING, "Cannot render document XML; expected hit of type " +
- "com.yahoo.storage.searcher.Document[Remove]Hit, got " + hit.getClass().getName() +
- ". Is there another backend searcher present?");
- }
- }
-
- private void writeDocumentRemoveHit(Writer writer, DocumentRemoveHit remove) throws IOException {
- writer.write("<remove documentid=\"");
- writer.write(XML.xmlEscape(remove.getIdOfRemovedDoc().toString()));
- writer.write("\"/>\n");
- }
-
- @Override
- public void hitFooter(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException {
- }
-
- @Override
- public void noHits(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException {
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java
deleted file mode 100755
index 869a2f9317a..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java
+++ /dev/null
@@ -1,508 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.storage.searcher;
-
-import com.google.inject.Inject;
-import com.yahoo.cloud.config.ClusterListConfig;
-import com.yahoo.cloud.config.SlobroksConfig;
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.document.config.DocumentmanagerConfig;
-import com.yahoo.jdisc.Metric;
-import com.yahoo.processing.request.CompoundName;
-import com.yahoo.vespa.config.content.LoadTypeConfig;
-import com.yahoo.document.DataType;
-import com.yahoo.document.Document;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.Field;
-import com.yahoo.document.datatypes.FieldValue;
-import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
-import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply;
-import com.yahoo.feedapi.*;
-import com.yahoo.log.LogLevel;
-import com.yahoo.messagebus.Reply;
-import com.yahoo.search.Query;
-import com.yahoo.search.query.Properties;
-import com.yahoo.search.Result;
-import com.yahoo.search.Searcher;
-import com.yahoo.search.result.DefaultErrorHit;
-import com.yahoo.search.result.ErrorMessage;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.vespaclient.config.FeederConfig;
-
-import java.io.*;
-import java.util.*;
-import java.util.logging.Logger;
-import java.util.zip.GZIPInputStream;
-
-/**
- * Searcher component to make GET requests to a content cluster.
- * <p>
- * Document ID must be given either as 1 "id=docid" query parameter
- * for single-document GETs and 1-n "id[0]=docid_1&amp;id[1]=...&amp;id[n-1]=docid_n"
- * parameters for multi-document GETs.
- *
- * <p>
- * Standard gateway query parameters are implicitly supported:
- * priority, timeout, route
- *
- * <p>
- * The searcher also accepts the following (optional) query parameters:
- * headersonly=true|false (default: false)
- * For specifying whether or not to return only header fields.
- *
- * <p>
- * field=string (default: no parameter specified)
- * For getting a single document field.
- *
- * <p>
- * contenttype=string (default: no content type specified)
- * For specifiying the returned HTTP header content type for a returned
- * document field's content. field must also be specified.
- * @deprecated do not use
- */
-@SuppressWarnings("deprecation")
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class GetSearcher extends Searcher {
-
- private static final Logger log = Logger.getLogger(GetSearcher.class.getName());
-
- private static final CompoundName ID = new CompoundName("id");
- private static final CompoundName HEADERS_ONLY = new CompoundName("headersonly");
- private static final CompoundName POPULATE_HIT_FIELDS = new CompoundName("populatehitfields");
- private static final CompoundName FIELDSET = new CompoundName("fieldset");
- private static final CompoundName FIELD = new CompoundName("field");
- private static final CompoundName CONTENT_TYPE = new CompoundName("contenttype");
- private static final CompoundName TIEMOUT = new CompoundName("timeout");
-
- FeedContext context;
-
- private final long defaultTimeoutMillis;
-
- private class GetResponse implements SharedSender.ResultCallback {
-
- /**
- * We have to maintain the same ordering of results as that
- * given in the request. Do this by remembering the index of
- * each requested document ID.
- */
- private Map<String, Integer> ordering;
- private List<DocumentHit> documentHits = new ArrayList<>();
- private List<DefaultErrorHit> errorHits = new ArrayList<>();
- private List<Reply> replies = new ArrayList<>();
- private final SharedSender.Pending pendingNumber = new SharedSender.Pending();
-
- public GetResponse(List<String> documentIds) {
- ordering = new HashMap<>(documentIds.size());
- for (int i = 0; i < documentIds.size(); ++i) {
- ordering.put(documentIds.get(i), i);
- }
- }
-
- public boolean isAborted() {
- return false;
- }
-
- private String stackTraceFromException(Exception e) {
- StringWriter sw = new StringWriter();
- PrintWriter ps = new PrintWriter(sw);
- e.printStackTrace(ps);
- ps.flush();
- return sw.toString();
- }
-
- public boolean handleReply(Reply reply) {
- if ((reply.getTrace().getLevel() > 0) && log.isLoggable(LogLevel.DEBUG)) {
- String str = reply.getTrace().toString();
- log.log(LogLevel.DEBUG, str);
- }
- replies.add(reply);
- return true;
- }
- public SharedSender.Pending getPending() { return pendingNumber; }
-
- private void processReplies() {
- for (Reply reply : replies) {
- processReply(reply);
- }
- }
- private void processReply(Reply reply) {
- if (!reply.hasErrors()) {
- try {
- addDocumentHit(reply);
- } catch (Exception e) {
- String msg = "Got exception of type " + e.getClass().getName()
- + " during document deserialization: " + e.getMessage();
- errorHits.add(new DefaultErrorHit("GetSearcher", ErrorMessage.createInternalServerError(msg)));
- log.log(LogLevel.DEBUG, "Got exception during document deserialization: " + stackTraceFromException(e));
- }
- } else {
- errorHits.add(new DefaultErrorHit("GetSearcher", new MessageBusErrorMessage(
- reply.getError(0).getCode(), 0, reply.getError(0).getMessage())));
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Received error reply with message " + reply.getError(0).getMessage());
- }
- }
- }
-
- private void addDocumentHit(Reply reply) {
- Document doc = ((GetDocumentReply)reply).getDocument();
- GetDocumentMessage msg = (GetDocumentMessage)reply.getMessage();
- Integer index = ordering.get(msg.getDocumentId().toString());
- if (index == null) { // Shouldn't happen
- throw new IllegalStateException("Received GetDocumentReply for unknown document: "
- + doc.getId().toString());
- }
- if (doc != null) {
- documentHits.add(new DocumentHit(doc, index));
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Received GetDocumentReply for "
- + doc.getId().toString());
- }
- } else {
- // Don't add a hit for documents that can't be found
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Received empty (not found) GetDocumentReply for "
- + msg.getDocumentId().toString());
- }
- }
- }
-
- private class IndexComparator implements Comparator<DocumentHit> {
- public int compare(DocumentHit o1, DocumentHit o2) {
- return o1.getIndex() - o2.getIndex();
- }
- }
-
- public void addHitsToResult(Result result, boolean populateHitFields) {
- for (DefaultErrorHit hit : errorHits) {
- result.hits().add(hit);
- }
- // Sort document hits according to their request index
- Collections.sort(documentHits, new IndexComparator());
- for (DocumentHit hit : documentHits) {
- if (populateHitFields) {
- hit.populateHitFields();
- }
- result.hits().add(hit);
- }
- result.setTotalHitCount(documentHits.size());
- }
-
- public List<DocumentHit> getDocumentHits() {
- return documentHits;
- }
-
- public List<DefaultErrorHit> getErrorHits() {
- return errorHits;
- }
- }
-
- @Inject
- public GetSearcher(FeederConfig feederConfig,
- LoadTypeConfig loadTypeConfig,
- DocumentmanagerConfig documentmanagerConfig,
- SlobroksConfig slobroksConfig,
- ClusterListConfig clusterListConfig,
- Metric metric)
- throws Exception
- {
- this(FeedContext.getInstance(feederConfig, loadTypeConfig,
- documentmanagerConfig, slobroksConfig,
- clusterListConfig, metric),
- (long)(feederConfig.timeout() * 1000));
- }
-
- GetSearcher(FeedContext context) throws Exception {
- this.context = context;
- this.defaultTimeoutMillis = context.getPropertyProcessor().getDefaultTimeoutMillis();
- }
-
- GetSearcher(FeedContext context, long defaultTimeoutMillis) throws Exception {
- this.context = context;
- this.defaultTimeoutMillis = defaultTimeoutMillis;
- }
-
- private static void postValidateDocumentIdParameters(Properties properties, int arrayIdsFound) throws Exception {
- for (Map.Entry<String, Object> kv : properties.listProperties().entrySet()) {
- if (!kv.getKey().startsWith("id[")) {
- continue;
- }
- if (!kv.getKey().endsWith("]")) {
- throw new IllegalArgumentException("Malformed document ID array parameter");
- }
- String indexStr = kv.getKey().substring(3, kv.getKey().length() - 1);
- int idx = Integer.parseInt(indexStr);
- if (idx >= arrayIdsFound) {
- throw new IllegalArgumentException("query contains document ID array " +
- "that is not zero-based and/or linearly increasing");
- }
- }
- }
-
- private List<String> getDocumentIds(Query query) throws Exception {
- Properties properties = query.properties();
- List<String> docIds = new ArrayList<>();
-
- // First check for regular "id=XX" syntax. If found, return vector with that
- // document id only
- String singleId = properties.getString(ID);
-
- int index = 0;
- if (singleId != null) {
- docIds.add(singleId);
- } else {
- // Check for id[0]=XX&id[1]=YY...id[n]=ZZ syntax. Indices always start
- // at 0 and are always increased by 1.
- while (true) {
- String docId = properties.getString("id[" + index + "]");
- if (docId == null) {
- break;
- }
- docIds.add(docId);
- ++index;
- }
- postValidateDocumentIdParameters(properties, index);
- }
-
- handleData(query.getHttpRequest(), docIds);
- return docIds;
- }
-
- private void handleData(HttpRequest request, List<String> docIds) throws IOException {
- if (request.getData() != null) {
- InputStream input;
- if ("gzip".equals(request.getHeader("Content-Encoding"))) {
- input = new GZIPInputStream(request.getData());
- } else {
- input = request.getData();
- }
- InputStreamReader reader = new InputStreamReader(input, "UTF-8");
- BufferedReader lineReader = new BufferedReader(reader);
- String line;
- while ((line = lineReader.readLine()) != null) {
- docIds.add(line);
- }
- }
- }
-
- private void handleFieldFiltering(GetResponse response, Result result,
- String fieldName, String contentType,
- boolean headersOnly) {
-
- if (response.getDocumentHits().isEmpty()) {
- result.hits().addError(ErrorMessage.createNotFound(
- "Document not found, could not return field '" + fieldName + "'"));
- return;
- }
-
- if (result.hits().getErrorHit() == null) {
- Document doc = response.getDocumentHits().get(0).getDocument();
- Field field = doc.getDataType().getField(fieldName);
- boolean wrapXml = false;
-
- if (field == null) {
- result.hits().addError(ErrorMessage.createIllegalQuery(
- "Field '" + fieldName + "' not found in document type"));
- return;
- }
- FieldValue value = doc.getFieldValue(field);
- // If the field exists but hasn't been set in this document, the
- // content will be null. We treat this as an error.
- if (value == null) {
- if (!field.isHeader() && headersOnly) {
- // TODO(vekterli): make this work with field sets as well.
- result.hits().addError(ErrorMessage.createInvalidQueryParameter(
- "Field '" + fieldName + "' is located in document body, but headersonly "
- + "prevents it from being retrieved in " + doc.getId().toString()));
- } else {
- result.hits().addError(ErrorMessage.createNotFound(
- "Field '" + fieldName + "' found in document type, but had "
- + "no content in " + doc.getId().toString()));
- }
- return;
- }
- String encoding = null;
- if (field.getDataType() == DataType.RAW) {
- if (contentType == null) {
- contentType = "application/octet-stream";
- }
- encoding = "ISO-8859-1";
- } else {
- // By default, return field wrapped in a blanket of vespa XML
- contentType = "text/xml";
- wrapXml = true;
- }
- if (encoding == null) {
- // Encoding doesn't matter for binary content, since we're always
- // writing directly to the byte buffer and not through a charset
- // encoder. Presumably, the client is intelligent enough to not
- // attempt to UTF-8 decode binary data.
- encoding = "UTF-8";
- }
- // Add hit now that we know there aren't any field errors. Otherwise,
- // there would be both an error hit and a document hit in the result
- response.addHitsToResult(result, false);
- // Override Vespa XML template
- result.getTemplating().setTemplates(new DocumentFieldTemplate(field, contentType, encoding, wrapXml));
- }
- // else: return with error hit, invoking regular Vespa XML error template
- }
-
- private void validateParameters(String fieldName, String contentType,
- List<String> documentIds) {
- // Content-type only makes sense for single document queries with a field
- // set
- if (contentType != null) {
- if (documentIds.size() > 1) {
- throw new IllegalArgumentException(
- "contenttype parameter only valid for single document id query");
- }
- if (fieldName == null) {
- throw new IllegalArgumentException(
- "contenttype set without document field being specified");
- }
- }
- if (fieldName != null && documentIds.size() > 1) {
- throw new IllegalArgumentException(
- "Field only valid for single document id query");
- }
- }
-
- // For unit testing
- protected MessagePropertyProcessor getMessagePropertyProcessor() {
- return context.getPropertyProcessor();
- }
-
- private void doGetDocuments(Query query, Result result, List<String> documentIds) {
- GetResponse response = new GetResponse(documentIds);
- Properties properties = query.properties();
-
- boolean headersOnly = properties.getBoolean(HEADERS_ONLY, false);
- boolean populateHitFields = properties.getBoolean(POPULATE_HIT_FIELDS, false);
- String fieldSet = properties.getString(FIELDSET);
- String fieldName = properties.getString(FIELD);
- String contentType = properties.getString(CONTENT_TYPE);
- long timeoutMillis = properties.getString(TIEMOUT) != null ? query.getTimeout() : defaultTimeoutMillis;
-
- if (fieldSet == null) {
- fieldSet = headersOnly ? "[header]" : "[all]";
- }
-
- validateParameters(fieldName, contentType, documentIds);
-
- MessagePropertyProcessor.PropertySetter propertySetter;
- propertySetter = context.getPropertyProcessor().buildPropertySetter(query.getHttpRequest());
-
- SingleSender sender = new SingleSender(response, context.getSharedSender(propertySetter.getRoute().toString()));
- sender.addMessageProcessor(propertySetter);
-
- sendDocumentGetMessages(documentIds, fieldSet, sender);
- // Twiddle thumbs until we've received a reply for all documents
- sender.done();
- boolean completed = sender.waitForPending(timeoutMillis);
- if ( ! completed) {
- result.hits().addError(ErrorMessage.createTimeout(
- "Timed out after waiting "+timeoutMillis+" ms for responses"));
- }
- response.processReplies();
- if (fieldName != null) {
- handleFieldFiltering(response, result, fieldName, contentType, headersOnly);
- } else {
- response.addHitsToResult(result, populateHitFields);
- }
- }
-
- private void sendDocumentGetMessages(List<String> documentIds, String fieldSet, SingleSender sender) {
- for (String docIdStr : documentIds) {
- DocumentId docId = new DocumentId(docIdStr);
- GetDocumentMessage getMsg = new GetDocumentMessage(docId, fieldSet);
-
- sender.send(getMsg);
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Sent GetDocumentMessage for "
- + docId.toString());
- }
- }
- }
-
- boolean verifyBackendDocumentHitsOnly(Result result) {
- if (result.hits().size() != 0) {
- log.log(LogLevel.DEBUG, "Result had hits after being sent down");
- for (int i = 0; i < result.hits().size(); ++i) {
- if (!(result.hits().get(i) instanceof DocumentHit)) {
- log.log(LogLevel.WARNING, "Got hit from backend searcher which was "
- + "not a com.yahoo.storage.searcher.DocumentHit instance: "
- + result.hits().get(i).getClass().getName());
- return false;
- }
- }
- }
- return true;
- }
-
- @Override
- public Result search(Query query, Execution execution) {
- // Pass through to next searcher
- Result result = execution.search(query);
-
- List<String> documentIds;
- try {
- documentIds = getDocumentIds(query);
- } catch (Exception e) {
- setOutputFormat(query, result);
- result.hits().addError(ErrorMessage.createIllegalQuery(e.getClass().getName() + ": " + e.getMessage()));
- return result;
- }
- // Early-out for pass-through queries
- if (documentIds.isEmpty()) {
- return result;
- }
- // Make sure we don't try to combine non-document hits and document hits
- // in the same result set.
- if (!verifyBackendDocumentHitsOnly(result)) {
- result = new Result(query); // Don't include unknown hits
- setOutputFormat(query, result);
- result.hits().addError(ErrorMessage.createInternalServerError(
- "A backend searcher to com.yahoo.storage.searcher.GetSearcher " +
- "returned a hit that was not an instance of com.yahoo.storage.searcher.DocumentHit. " +
- "Only DocumentHit instances are supported in the backend hit result set when doing " +
- "queries that contain document identifier sets recognised by the Get Searcher."));
- return result;
- }
- setOutputFormat(query, result);
- // Do not propagate exceptions back up, as we want to have all errors
- // be reported using the proper template
- try {
- doGetDocuments(query, result, documentIds);
- query.setHits(result.hits().size());
- } catch (IllegalArgumentException e) {
- result.hits().addError(ErrorMessage.createIllegalQuery(e.getClass().getName() + ": " + e.getMessage()));
- } catch (Exception e) {
- result.hits().addError(ErrorMessage.createUnspecifiedError(e.getClass().getName() + ": " + e.getMessage()));
- }
-
- return result;
- }
-
- private static final CompoundName formatShortcut = new CompoundName("format");
- private static final CompoundName format = new CompoundName("presentation.format");
-
- /**
- * Use custom XML output format unless the default JSON renderer is specified in the request.
- */
- @SuppressWarnings("deprecation")
- static void setOutputFormat(Query query, Result result) {
- if (getRequestProperty(formatShortcut, "", query).equals("JsonRenderer")) return;
- if (getRequestProperty(format, "", query).equals("JsonRenderer")) return;
- if (getRequestProperty(formatShortcut, "", query).equals("json")) return;
- if (getRequestProperty(format, "", query).equals("json")) return;
- result.getTemplating().setTemplates(new DocumentXMLTemplate());
- }
-
- private static String getRequestProperty(CompoundName propertyName, String defaultValue, Query query) {
- String propertyValue = query.getHttpRequest().getProperty(propertyName.toString());
- if (propertyValue == null) return defaultValue;
- return propertyValue;
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java
deleted file mode 100755
index ad93645e55e..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.storage.searcher;
-
-import com.yahoo.search.result.ErrorMessage;
-
-/**
- * Simple ErrorMessage extension that includes a message bus error code, not
- * just the searcher error code (which isn't very useful for a Vespa XML consumer)
- */
-@Deprecated // OK
-// TODO: Remove on Vespa 7
-public class MessageBusErrorMessage extends ErrorMessage {
-
- private int mbusCode;
-
- public MessageBusErrorMessage(int mbusCode, int qrsCode, String message) {
- super(qrsCode, message);
- this.mbusCode = mbusCode;
- }
-
- public MessageBusErrorMessage(int mbusCode, int qrsCode, String message, String detailedMessage) {
- super(qrsCode, message, detailedMessage);
- this.mbusCode = mbusCode;
- }
-
- public MessageBusErrorMessage(int mbusCode, int qrsCode, String message, String detailedMessage, Throwable cause) {
- super(qrsCode, message, detailedMessage, cause);
- this.mbusCode = mbusCode;
- }
-
- public int getMessageBusCode() {
- return mbusCode;
- }
-
- public void setMessageBusCode(int code) {
- this.mbusCode = code;
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java
deleted file mode 100644
index e330b76fa4b..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java
+++ /dev/null
@@ -1,210 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.storage.searcher;
-
-import com.yahoo.cloud.config.ClusterListConfig;
-import com.yahoo.cloud.config.SlobroksConfig;
-import com.yahoo.document.config.DocumentmanagerConfig;
-import com.yahoo.vespa.config.content.LoadTypeConfig;
-import com.yahoo.component.ComponentId;
-import com.yahoo.component.ComponentSpecification;
-import com.yahoo.document.Document;
-import com.yahoo.document.DocumentId;
-import com.yahoo.documentapi.*;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
-import com.yahoo.feedapi.FeedContext;
-import com.yahoo.feedapi.MessagePropertyProcessor;
-import com.yahoo.jdisc.Metric;
-import com.yahoo.messagebus.StaticThrottlePolicy;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.Searcher;
-import com.yahoo.search.rendering.RendererRegistry;
-import com.yahoo.search.result.ErrorMessage;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.vdslib.VisitorOrdering;
-import com.yahoo.vespaclient.ClusterDef;
-import com.yahoo.vespaclient.config.FeederConfig;
-
-/**
- * A searcher that allows you to iterate through a storage cluster using visiting.
- *
- * @deprecated do not use
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class VisitSearcher extends Searcher {
-
- public static final String VISITOR_CONTINUATION_TOKEN_FIELDNAME = "visitorContinuationToken";
- FeedContext context;
-
- public VisitSearcher(FeederConfig feederConfig,
- LoadTypeConfig loadTypeConfig,
- DocumentmanagerConfig documentmanagerConfig,
- SlobroksConfig slobroksConfig,
- ClusterListConfig clusterListConfig,
- Metric metric)
- throws Exception
- {
- this(FeedContext.getInstance(feederConfig, loadTypeConfig,
- documentmanagerConfig, slobroksConfig,
- clusterListConfig, metric));
- }
-
- VisitSearcher(FeedContext context) throws Exception {
- this.context = context;
- }
-
- class HitDataHandler extends DumpVisitorDataHandler {
- private final Result result;
- private final boolean populateHits;
- private final Object monitor = new Object();
-
- HitDataHandler(Result result, boolean populateHits) {
- this.result = result;
- this.populateHits = populateHits;
- }
-
- @Override
- public void onDocument(Document document, long l) {
- final DocumentHit hit = new DocumentHit(document, 0);
- if (populateHits) {
- hit.populateHitFields();
- }
- synchronized (monitor) {
- result.hits().add(hit);
- }
- }
-
- @Override
- public void onRemove(DocumentId documentId) {
- final DocumentRemoveHit hit = new DocumentRemoveHit(documentId);
- synchronized (monitor) {
- result.hits().add(hit);
- }
- }
- }
-
- public VisitorParameters getVisitorParameters(Query query, Result result) throws Exception {
- String documentSelection = query.properties().getString("visit.selection");
- if (documentSelection == null) {
- documentSelection = "";
- }
-
- VisitorParameters params = new VisitorParameters(documentSelection);
- params.setMaxBucketsPerVisitor(query.properties().getInteger("visit.maxBucketsPerVisitor", 1));
- params.setMaxPending(query.properties().getInteger("visit.maxPendingMessagesPerVisitor", 32));
- params.setMaxFirstPassHits(query.properties().getInteger("visit.approxMaxDocs", 1));
- params.setMaxTotalHits(query.properties().getInteger("visit.approxMaxDocs", 1));
- params.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount(
- query.properties().getInteger("visit.maxPendingVisitors", 1)));
- params.setToTimestamp(query.properties().getLong("visit.toTimestamp", 0L));
- params.setFromTimestamp(query.properties().getLong("visit.fromTimestamp", 0L));
-
- String pri = query.properties().getString("visit.priority");
- if (pri != null) {
- params.setPriority(DocumentProtocol.Priority.valueOf(pri));
- }
-
- if (query.properties().getBoolean("visit.visitInconsistentBuckets")) {
- params.visitInconsistentBuckets(true);
- }
-
- String ordering = query.properties().getString("visit.order");
- if (!"ascending".equalsIgnoreCase(ordering)) {
- params.setVisitorOrdering(VisitorOrdering.ASCENDING);
- } else {
- params.setVisitorOrdering(VisitorOrdering.DESCENDING);
- }
-
- String remoteCluster = query.properties().getString("visit.dataHandler");
- if (remoteCluster != null) {
- params.setRemoteDataHandler(remoteCluster);
- } else {
- params.setLocalDataHandler(new HitDataHandler(
- result, query.properties().getBoolean("populatehitfields", false)));
- }
-
- String fieldSet = query.properties().getString("visit.fieldSet");
- if (fieldSet != null) {
- params.fieldSet(fieldSet);
- }
-
- String continuation = query.properties().getString("visit.continuation");
- if (continuation != null) {
- params.setResumeToken(ContinuationHit.getToken(continuation));
- }
-
- params.setVisitRemoves(query.properties().getBoolean("visit.visitRemoves"));
-
- MessagePropertyProcessor.PropertySetter propertySetter;
- propertySetter = context.getPropertyProcessor().buildPropertySetter(query.getHttpRequest());
-
- propertySetter.process(params);
-
- if (context.getClusterList().getStorageClusters().size() == 0) {
- throw new IllegalArgumentException("No content clusters have been defined");
- }
-
- String route = query.properties().getString("visit.cluster");
- ClusterDef found = null;
- if (route != null) {
- String names = "";
- for (ClusterDef c : context.getClusterList().getStorageClusters()) {
- if (c.getName().equals(route)) {
- found = c;
- }
- if (!names.isEmpty()) {
- names += ", ";
- }
- names += c.getName();
- }
- if (found == null) {
- throw new IllegalArgumentException("Your vespa cluster contains the storage clusters " + names + ", not " + route + ". Please select a valid vespa cluster.");
- }
- } else if (context.getClusterList().getStorageClusters().size() == 1) {
- found = context.getClusterList().getStorageClusters().get(0);
- } else {
- throw new IllegalArgumentException("Multiple content clusters are defined, select one using the \"visit.cluster\" option");
- }
-
- params.setRoute("[Storage:cluster=" + found.getName() + ";clusterconfigid=" + found.getConfigId() + "]");
- return params;
- }
-
- @Override
- public Result search(Query query, Execution execution) {
- Result result = execution.search(query);
-
- VisitorParameters parameters;
-
- try {
- parameters = getVisitorParameters(query, result);
- } catch (Exception e) {
- return new Result(query, ErrorMessage.createBadRequest("Illegal parameters: " + e.toString()));
- }
-
- if (parameters != null) {
- VisitorSession session = context.getSessionFactory().createVisitorSession(parameters);
-
- try {
- if (!session.waitUntilDone(query.getTimeout())) {
- return new Result(query, ErrorMessage.createTimeout("Visitor timed out"));
- }
-
- ProgressToken token = session.getProgress();
- if (!token.isFinished()) {
- final ContinuationHit continuation = new ContinuationHit(token);
- result.hits().setField(VISITOR_CONTINUATION_TOKEN_FIELDNAME, continuation.getValue());
- }
- } catch (InterruptedException e) {
- } finally {
- session.destroy();
- }
- }
-
- GetSearcher.setOutputFormat(query, result);
- query.setHits(result.hits().size());
- return result;
- }
-
-}
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java
deleted file mode 100644
index 581afb0b1c8..00000000000
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java
+++ /dev/null
@@ -1,6 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-// TODO: This implements the old, deprecated document http API. Remove this package on Vespa 7.
-@ExportPackage
-package com.yahoo.storage.searcher;
-
-import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java
index b4067c3fe78..6bca3637297 100644
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java
@@ -32,4 +32,5 @@ public class DocumentApiApplicationTest {
socket.close();
return port;
}
+
}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore b/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore
deleted file mode 100644
index e69de29bb2d..00000000000
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore
+++ /dev/null
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java
deleted file mode 100755
index 2c99ec194c3..00000000000
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java
+++ /dev/null
@@ -1,1064 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.feedhandler;
-
-import com.yahoo.component.ComponentId;
-import com.yahoo.component.provider.ComponentRegistry;
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.docproc.CallStack;
-import com.yahoo.jdisc.HeaderFields;
-import com.yahoo.messagebus.*;
-import com.yahoo.vespa.config.content.LoadTypeConfig;
-import com.yahoo.container.Container;
-import com.yahoo.docproc.*;
-import com.yahoo.docproc.jdisc.DocumentProcessingHandler;
-import com.yahoo.docproc.jdisc.DocumentProcessingHandlerParameters;
-import com.yahoo.document.*;
-import com.yahoo.document.datatypes.IntegerFieldValue;
-import com.yahoo.documentapi.messagebus.loadtypes.LoadType;
-import com.yahoo.documentapi.messagebus.protocol.*;
-import com.yahoo.feedapi.DummySessionFactory;
-import com.yahoo.feedapi.FeedContext;
-import com.yahoo.feedapi.MessagePropertyProcessor;
-import com.yahoo.jdisc.handler.RequestHandler;
-import com.yahoo.messagebus.routing.Route;
-import com.yahoo.vespaclient.ClusterDef;
-import com.yahoo.vespaclient.ClusterList;
-import com.yahoo.vespaclient.config.FeederConfig;
-import org.junit.After;
-import org.junit.Test;
-
-import java.io.*;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Executors;
-import java.util.logging.Logger;
-import java.util.zip.GZIPOutputStream;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-@SuppressWarnings("deprecation") // VespaFeedHandler classes are going away on Vespa 7
-public class VespaFeedHandlerTestCase {
-
- private VespaFeedHandler feedHandler;
- private VespaFeedHandlerRemove removeHandler;
- private VespaFeedHandlerStatus statusHandler;
- private VespaFeedHandlerRemoveLocation removeLocationHandler;
- private FeedContext context;
-
- private DummySessionFactory factory;
- private final String xmlFilesPath = "src/test/files/feedhandler/";
-
- public void setup(com.yahoo.messagebus.Error e, LoadTypeConfig loadTypeCfg,
- boolean autoReply,
- DummySessionFactory.ReplyFactory autoReplyFactory) throws Exception {
- DocumentTypeManager docMan = new DocumentTypeManager();
- DocumentTypeManagerConfigurer.configure(docMan, "file:" + xmlFilesPath + "documentmanager.cfg");
-
- if (autoReply) {
- if (autoReplyFactory != null) {
- factory = DummySessionFactory.createWithAutoReplyFactory(autoReplyFactory);
- } else {
- factory = DummySessionFactory.createWithErrorAutoReply(e);
- }
- } else {
- factory = DummySessionFactory.createDefault();
- }
-
- context = new FeedContext(new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder()), loadTypeCfg), factory, docMan, new ClusterList(), new NullFeedMetric(true));
-
- Executor threadPool = Executors.newCachedThreadPool();
- feedHandler = new VespaFeedHandler(context, threadPool);
- removeHandler = new VespaFeedHandlerRemove(context, threadPool);
- statusHandler = new VespaFeedHandlerStatus(context, false, false, threadPool);
- removeLocationHandler = new VespaFeedHandlerRemoveLocation(context, threadPool);
-
- CallStack dpCallstack = new CallStack("bar");
- dpCallstack.addLast(new TestDocProc());
- dpCallstack.addLast(new TestLaterDocProc());
-
- DocprocService myservice = new DocprocService("bar");
- myservice.setCallStack(dpCallstack);
- myservice.setInService(true);
-
- ComponentRegistry<DocprocService> registry = new ComponentRegistry<DocprocService>();
- registry.register(new ComponentId(myservice.getName()), myservice);
-
- DocumentProcessingHandler handler = new DocumentProcessingHandler(registry,
- new ComponentRegistry<>(), new ComponentRegistry<>(),
- new DocumentProcessingHandlerParameters());
-
- Container container = Container.get();
- ComponentRegistry<RequestHandler> requestHandlerComponentRegistry = new ComponentRegistry<>();
- requestHandlerComponentRegistry.register(new ComponentId(DocumentProcessingHandler.class.getName()), handler);
- container.setRequestHandlerRegistry(requestHandlerComponentRegistry);
- }
-
- public void setup(com.yahoo.messagebus.Error e) throws Exception {
- setup(e, new LoadTypeConfig(new LoadTypeConfig.Builder()), true, null);
- }
-
- public void setupWithReplyFactory(DummySessionFactory.ReplyFactory autoReplyFactory) throws Exception {
- setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder()), true, autoReplyFactory);
- }
-
- public void setup() throws Exception {
- setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder()), false, null);
- }
-
- @After
- public void resetContainer() {
- Container.resetInstance();
- }
-
-
- @Test
- public void testLoadTypes() throws Exception {
- List<LoadTypeConfig.Type.Builder> typeBuilder = new ArrayList<>();
- typeBuilder.add(new LoadTypeConfig.Type.Builder().id(1234).name("foo").priority("VERY_LOW"));
- typeBuilder.add(new LoadTypeConfig.Type.Builder().id(4567).name("bar").priority("NORMAL_3"));
-
- setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder().type(typeBuilder)), true, null);
-
- {
- Result res = testRequest(HttpRequest.createTestRequest("remove?id=doc:test:removeme&loadtype=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
- assertEquals(1, res.messages.size());
-
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
- DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
- assertEquals("doc:test:removeme", d.toString());
- assertEquals(new LoadType(1234, "foo", DocumentProtocol.Priority.VERY_LOW), ((DocumentMessage)m).getLoadType());
- assertEquals(DocumentProtocol.Priority.VERY_LOW, ((DocumentMessage)m).getPriority());
- }
- }
-
- @Test
- public void testPostXML() throws Exception {
- setup(null);
- Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?");
-
- assertEquals(2, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10a", d.toString());
- }
- {
- Message m = res.messages.get(1);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10b", d.toString());
- }
-
- assertTrue(res.output.contains("count=\"2\""));
- assertTrue(res.error == null);
- }
-
- @Test
- public void testPostXMLAsync() throws Exception {
- setup();
- Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?asynchronous=true");
-
- assertEquals(2, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10a", d.toString());
- }
- {
- Message m = res.messages.get(1);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10b", d.toString());
- }
-
- // Should not have metrics at this point.
- assertTrue(!res.output.contains("count=\"2\""));
- assertTrue(res.error == null);
- }
-
-
- @Test
- public void testPostGZIPedXML() throws Exception {
- setup(null);
- Result res = testFeedGZIP(xmlFilesPath + "test10b.xml", "feed?");
-
- assertEquals(2, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10a", d.toString());
- }
- {
- Message m = res.messages.get(1);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10b", d.toString());
- }
-
- assertTrue(res.error == null);
- }
-
- @Test
- public void testDocProc() throws Exception {
- setup(null);
-
- Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar");
-
- assertEquals(2, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- Document d = ((PutDocumentMessage)m).getDocumentPut().getDocument();
-
- assertEquals("doc:news:http://news10a", d.getId().toString());
- assertEquals(new IntegerFieldValue(1234), d.getFieldValue("last_downloaded"));
- }
- {
- Message m = res.messages.get(1);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- Document d = ((PutDocumentMessage)m).getDocumentPut().getDocument();
-
- assertEquals("doc:news:http://news10b", d.getId().toString());
- assertEquals(new IntegerFieldValue(1234), d.getFieldValue("last_downloaded"));
- }
- }
-
- @Test
- public void testPostXMLVariousTypes() throws Exception {
- setup(null);
- Result res = testFeed(xmlFilesPath + "test10.xml", "feed?");
-
- assertEquals(5, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10a", d.toString());
- }
- {
- Message m = res.messages.get(1);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10b", d.toString());
- }
-
- {
- Message m = res.messages.get(2);
- assertEquals(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, m.getType());
- DocumentId d = ((UpdateDocumentMessage)m).getDocumentUpdate().getId();
- assertEquals("doc:news:http://news10c", d.toString());
- }
- {
- Message m = res.messages.get(3);
- assertEquals(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, m.getType());
- DocumentId d = ((UpdateDocumentMessage)m).getDocumentUpdate().getId();
- assertEquals("doc:news:http://news10d", d.toString());
- }
- {
- Message m = res.messages.get(4);
- assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
- DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
- assertEquals("doc:news:http://news10e", d.toString());
- }
-
- String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/");
-
- assertEquals("<result>\n" +
- "\n" +
- " <route name=\"default\">\n" +
- " <total>\n" +
- " <latency count=\"5\"/>\n" +
- " <count count=\"5\"/>\n" +
- " </total>\n" +
- " <putdocument>\n" +
- " <latency count=\"2\"/>\n" +
- " <count count=\"2\"/>\n" +
- " </putdocument>\n" +
- " <updatedocument>\n" +
- " <latency count=\"2\"/>\n" +
- " <count count=\"2\"/>\n" +
- " </updatedocument>\n" +
- " <removedocument>\n" +
- " <latency count=\"1\"/>\n" +
- " <count count=\"1\"/>\n" +
- " </removedocument>\n" +
- " </route>\n" +
- "\n" +
- "</result>\n", val);
- }
-
- @Test
- public void testStatusPage() throws Exception {
- setup(null);
-
- testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar");
- testFeed(xmlFilesPath + "test10.xml", "feed?");
- testFeed(xmlFilesPath + "test10.xml", "feed?route=storage");
- testFeed(xmlFilesPath + "test_removes", "remove?");
-
- assertEquals(2, factory.sessionsCreated());
- Result res = testRequest(HttpRequest.createTestRequest("feedstatus?", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
-
- String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/");
- val = val.replaceAll("to=\"[0-9]*\"", "to=\"0\"");
-
- assertEquals("<status>\n" +
- "\n" +
- " <snapshot name=\"Total metrics from start until current time\" from=\"0\" to=\"0\" period=\"0\">\n" +
- " <routes>\n" +
- " <route name=\"total\">\n" +
- " <total>\n" +
- " <latency count=\"14\"/>\n" +
- " <count count=\"14\"/>\n" +
- " </total>\n" +
- " <putdocument>\n" +
- " <latency count=\"6\"/>\n" +
- " <count count=\"6\"/>\n" +
- " </putdocument>\n" +
- " <updatedocument>\n" +
- " <latency count=\"4\"/>\n" +
- " <count count=\"4\"/>\n" +
- " </updatedocument>\n" +
- " <removedocument>\n" +
- " <latency count=\"4\"/>\n" +
- " <count count=\"4\"/>\n" +
- " </removedocument>\n" +
- " </route>\n" +
- " <route name=\"default\">\n" +
- " <total>\n" +
- " <latency count=\"9\"/>\n" +
- " <count count=\"9\"/>\n" +
- " </total>\n" +
- " <putdocument>\n" +
- " <latency count=\"4\"/>\n" +
- " <count count=\"4\"/>\n" +
- " </putdocument>\n" +
- " <updatedocument>\n" +
- " <latency count=\"2\"/>\n" +
- " <count count=\"2\"/>\n" +
- " </updatedocument>\n" +
- " <removedocument>\n" +
- " <latency count=\"3\"/>\n" +
- " <count count=\"3\"/>\n" +
- " </removedocument>\n" +
- " </route>\n" +
- " <route name=\"storage\">\n" +
- " <total>\n" +
- " <latency count=\"5\"/>\n" +
- " <count count=\"5\"/>\n" +
- " </total>\n" +
- " <putdocument>\n" +
- " <latency count=\"2\"/>\n" +
- " <count count=\"2\"/>\n" +
- " </putdocument>\n" +
- " <updatedocument>\n" +
- " <latency count=\"2\"/>\n" +
- " <count count=\"2\"/>\n" +
- " </updatedocument>\n" +
- " <removedocument>\n" +
- " <latency count=\"1\"/>\n" +
- " <count count=\"1\"/>\n" +
- " </removedocument>\n" +
- " </route>\n" +
- " </routes>\n" +
- " </snapshot>\n" +
- "\n" +
- "</status>\n", val);
- }
-
- @Test
- public void testStatusPage2() throws Exception {
- setup(null);
-
- testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar");
- testFeed(xmlFilesPath + "test10.xml", "feed?");
- testFeed(xmlFilesPath + "test10.xml", "feed?route=storage");
- testFeed(xmlFilesPath + "test_removes", "remove?");
-
- assertEquals(2, factory.sessionsCreated());
- Result res = testRequest(HttpRequest.createTestRequest("feed?status", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
-
- String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/");
- val = val.replaceAll("to=\"[0-9]*\"", "to=\"0\"");
-
- assertEquals("<status>\n" +
- "\n" +
- " <routes>\n" +
- " <route name=\"total\" description=\"Messages sent to all routes\">\n" +
- " <total description=\"All kinds of messages sent to the given route\">\n" +
- " <latency count=\"14\"/>\n" +
- " <count count=\"14\"/>\n" +
- " <ignored count=\"0\"/>\n" +
- " </total>\n" +
- " <putdocument>\n" +
- " <latency count=\"6\"/>\n" +
- " <count count=\"6\"/>\n" +
- " <ignored count=\"0\"/>\n" +
- " </putdocument>\n" +
- " <updatedocument>\n" +
- " <latency count=\"4\"/>\n" +
- " <count count=\"4\"/>\n" +
- " <ignored count=\"0\"/>\n" +
- " </updatedocument>\n" +
- " <removedocument>\n" +
- " <latency count=\"4\"/>\n" +
- " <count count=\"4\"/>\n" +
- " <ignored count=\"0\"/>\n" +
- " </removedocument>\n" +
- " </route>\n" +
- " <route name=\"default\" description=\"Messages sent to the named route\">\n" +
- " <total description=\"All kinds of messages sent to the given route\">\n" +
- " <latency count=\"9\"/>\n" +
- " <count count=\"9\"/>\n" +
- " <ignored count=\"0\"/>\n" +
- " </total>\n" +
- " <putdocument>\n" +
- " <latency count=\"4\"/>\n" +
- " <count count=\"4\"/>\n" +
- " <ignored count=\"0\"/>\n" +
- " </putdocument>\n" +
- " <updatedocument>\n" +
- " <latency count=\"2\"/>\n" +
- " <count count=\"2\"/>\n" +
- " <ignored count=\"0\"/>\n" +
- " </updatedocument>\n" +
- " <removedocument>\n" +
- " <latency count=\"3\"/>\n" +
- " <count count=\"3\"/>\n" +
- " <ignored count=\"0\"/>\n" +
- " </removedocument>\n" +
- " </route>\n" +
- " <route name=\"storage\" description=\"Messages sent to the named route\">\n" +
- " <total description=\"All kinds of messages sent to the given route\">\n" +
- " <latency count=\"5\"/>\n" +
- " <count count=\"5\"/>\n" +
- " <ignored count=\"0\"/>\n" +
- " </total>\n" +
- " <putdocument>\n" +
- " <latency count=\"2\"/>\n" +
- " <count count=\"2\"/>\n" +
- " <ignored count=\"0\"/>\n" +
- " </putdocument>\n" +
- " <updatedocument>\n" +
- " <latency count=\"2\"/>\n" +
- " <count count=\"2\"/>\n" +
- " <ignored count=\"0\"/>\n" +
- " </updatedocument>\n" +
- " <removedocument>\n" +
- " <latency count=\"1\"/>\n" +
- " <count count=\"1\"/>\n" +
- " <ignored count=\"0\"/>\n" +
- " </removedocument>\n" +
- " </route>\n" +
- " </routes>\n" +
- "\n" +
- "</status>\n", val);
- }
-
- @Test
- public void testMetricForIgnoredDocumentsIsIncreased() throws Exception {
- DummySessionFactory.ReplyFactory replyFactory = new DummySessionFactory.ReplyFactory() {
- @Override
- public Reply createReply(Message m) {
- return new DocumentIgnoredReply();
- }
- };
- setupWithReplyFactory(replyFactory);
- Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?");
- assertEquals(2, res.messages.size());
-
- String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/");
-
- assertEquals("<result>\n" +
- "\n" +
- " <route name=\"default\">\n" +
- " <total>\n" +
- " <ignored count=\"2\"/>\n" +
- " </total>\n" +
- " <putdocument>\n" +
- " <ignored count=\"2\"/>\n" +
- " </putdocument>\n" +
- " </route>\n" +
- "\n" +
- "</result>\n", val);
- }
-
- @Test
- public void testPostXMLWithMBusFailureAllowed() throws Exception {
- setup(new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_BUCKET_DELETED, "Hello world in <document>"));
- Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?abortonfeederror=false");
-
- assertEquals(2, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10a", d.toString());
- }
- {
- Message m = res.messages.get(1);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10b", d.toString());
- }
-
- String val = res.output.replaceAll("average=\"[0-9]*\" last=\"[0-9]*\" min=\"[0-9]*\" max=\"[0-9]*\" ", "");
- System.out.println(val);
-
- assertEquals("<result>\n" +
- "\n" +
- " <route name=\"default\">\n" +
- " <total>\n" +
- " <errors>\n" +
- " <error name=\"total\" count=\"2\"/>\n" +
- " <error name=\"BUCKET_DELETED\" count=\"2\"/>\n" +
- " </errors>\n" +
- " </total>\n" +
- " <putdocument>\n" +
- " <errors>\n" +
- " <error name=\"total\" count=\"2\"/>\n" +
- " <error name=\"BUCKET_DELETED\" count=\"2\"/>\n" +
- " </errors>\n" +
- " </putdocument>\n" +
- " </route>\n\n" +
- " <errors count=\"2\">\n" +
- " <error message=\"PUT[doc:news:http://news10a] [BUCKET_DELETED] Hello world in &lt;document&gt;\"/>\n" +
- " <error message=\"PUT[doc:news:http://news10b] [BUCKET_DELETED] Hello world in &lt;document&gt;\"/>\n" +
- " </errors>\n" +
- "\n" +
- "</result>\n", val);
-
- assertTrue(res.error != null);
- assertTrue(res.errorCount > 0);
- }
-
- @Test
- public void testPostXMLWithMBusFailure() throws Exception {
- setup(new com.yahoo.messagebus.Error(32, "Hello world"));
- Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?");
-
- assertEquals(1, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10a", d.toString());
- }
-
- String val = res.output.replaceAll("average=\"[0-9]*\" last=\"[0-9]*\" min=\"[0-9]*\" max=\"[0-9]*\" ", "");
- assertEquals("<result>\n" +
- "\n" +
- " <route name=\"default\">\n" +
- " <total>\n" +
- " <errors>\n" +
- " <error name=\"total\" count=\"1\"/>\n" +
- " <error name=\"UNKNOWN(32)\" count=\"1\"/>\n" +
- " </errors>\n" +
- " </total>\n" +
- " <putdocument>\n" +
- " <errors>\n" +
- " <error name=\"total\" count=\"1\"/>\n" +
- " <error name=\"UNKNOWN(32)\" count=\"1\"/>\n" +
- " </errors>\n" +
- " </putdocument>\n" +
- " </route>\n\n" +
- " <errors count=\"1\">\n" +
- " <error message=\"PUT[doc:news:http://news10a] [UNKNOWN(32)] Hello world\"/>\n" +
- " </errors>\n" +
- "\n" +
- "</result>\n", val);
-
- assertTrue(res.error != null);
- assertTrue(res.errorCount > 0);
- }
-
- @Test
- public void testPostXMLWithIllegalDocId() throws Exception {
- setup(null);
- Result res = testFeed(xmlFilesPath + "test_bogus_docid.xml", "feed?");
-
- assertEquals(1, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10a", d.toString());
- }
- }
-
- @Test
- public void testPostXMLWithIllegalDocIdAllowFailure() throws Exception {
- setup(null);
- Result res = testFeed(xmlFilesPath + "test_bogus_docid.xml", "feed?abortondocumenterror=false");
-
- assertEquals(2, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10a", d.toString());
- }
-
- {
- Message m = res.messages.get(1);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:ok", d.toString());
- }
- }
-
- @Test
- public void testPostUnparseableXML() throws Exception {
- setup(null);
- Result res = testFeed(xmlFilesPath + "test_bogus_xml.xml", "feed?");
-
- assertEquals(1, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10a", d.toString());
- }
- }
-
- @Test
- public void testOverrides() throws Exception {
- setup(null);
- Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222&route=storage&priority=HIGH_2&totaltimeout=-1");
-
- assertEquals(2, res.messages.size());
-
- for (Message m : res.messages) {
- assertEquals(2222, m.getTimeRemaining());
- assertEquals(Route.parse("storage"), m.getRoute());
- assertEquals(DocumentProtocol.Priority.HIGH_2, ((DocumentMessage)m).getPriority());
- }
- }
-
- @Test
- public void testTimeoutWithNoUpperBound() throws Exception {
- setup(null);
- Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222&totaltimeout=-1");
-
- assertEquals(2, res.messages.size());
-
- for (Message m : res.messages) {
- assertEquals(2222, m.getTimeRemaining());
- }
- }
-
- @Test
- public void testTimeout() throws Exception {
- setup(null);
- Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222");
-
- assertEquals(2, res.messages.size());
-
- for (Message m : res.messages) {
- assertTrue(2222 >= m.getTimeRemaining());
- }
- }
-
- @Test
- public void testTotalTimeout() throws Exception {
- setup(null);
- Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?totaltimeout=2.222");
-
- assertEquals(2, res.messages.size());
-
- for (Message m : res.messages) {
- assertTrue(2222 >= m.getTimeRemaining());
- }
- }
-
- @Test
- public void testTotalTimeoutAndNormalTimeout() throws Exception {
- setup(null);
- Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?totaltimeout=1000&timeout=2.222");
-
- assertEquals(2, res.messages.size());
-
- for (Message m : res.messages) {
- assertEquals(2222, m.getTimeRemaining());
- }
- }
-
- @Test
- public void testBogusPriority() throws Exception {
- try {
- setup(null);
- Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2222&route=storage&priority=HIPSTER_DOOFUS");
- assertTrue(false);
- } catch (IllegalArgumentException e) {
- }
- }
-
- @Test
- public void testPostXMLWithIllegalDocIdFirst() throws Exception {
- setup(null);
- Result res = testFeed(xmlFilesPath + "test_bogus_docid_first.xml", "feed?");
-
- assertEquals(0, res.messages.size());
- }
-
- @Test
- public void testPostXMLWithIllegalDocIdFirstNoAbort() throws Exception {
- setup(null);
- Result res = testFeed(xmlFilesPath + "test_bogus_docid_first.xml", "feed?abortondocumenterror=false");
-
- assertEquals(1, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType());
- DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId();
- assertEquals("doc:news:http://news10a", d.toString());
- }
- }
-
- @Test
- public void testSimpleRemove() throws Exception {
- setup(null);
- Result res = testRequest(HttpRequest.createTestRequest("remove?id=doc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
- assertEquals(1, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
- DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
- assertEquals("doc:test:removeme", d.toString());
- }
- }
-
- @Test
- public void testRemoveUser() throws Exception {
- setup(null);
-
- context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage"));
- Result res = testRequest(HttpRequest.createTestRequest("removelocation?user=1234", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
- assertEquals(1, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType());
- String selection = ((RemoveLocationMessage)m).getDocumentSelection();
- assertEquals("storage", m.getRoute().toString());
- assertEquals("id.user=1234", selection);
- }
- }
-
- @Test
- public void testRemoveGroup() throws Exception {
- setup(null);
- context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage"));
- Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
- assertEquals(1, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType());
- String selection = ((RemoveLocationMessage)m).getDocumentSelection();
- assertEquals("storage", m.getRoute().toString());
- assertEquals("id.group=\"foo\"", selection);
- }
- }
-
- @Test
- public void testRemoveBadSyntax() throws Exception {
- setup(null);
- context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage"));
- Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo&user=12345", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
- assertEquals(0, res.messages.size());
- assertTrue(res.error.toString().contains("Exactly one of"));
- }
-
- @Test
- public void testRemoveGroupMultipleClusters() throws Exception {
- setup(null);
- context.getClusterList().getStorageClusters().add(new ClusterDef("storage1", "storage/cluster.storage1"));
- context.getClusterList().getStorageClusters().add(new ClusterDef("storage2", "storage/cluster.storage2"));
- Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
- assertEquals(0, res.messages.size());
- assertTrue(res.error.toString().contains("More than one"));
- }
-
- @Test
- public void testRemoveGroupNoClusters() throws Exception {
- setup(null);
- Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
- assertEquals(0, res.messages.size());
- assertTrue(res.error.toString().contains("No storage clusters"));
- }
-
- @Test
- public void testRemoveSelection() throws Exception {
- setup(null);
- context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage"));
- Result res = testRequest(HttpRequest.createTestRequest("removelocation?selection=id.user=1234", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
- assertEquals(1, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType());
- String selection = ((RemoveLocationMessage)m).getDocumentSelection();
- assertEquals("id.user=1234", selection);
- }
- }
-
- @Test
- public void testSimpleRemoveIndex() throws Exception {
- setup(null);
- Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=doc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
- assertEquals(1, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
- DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
- assertEquals("doc:test:removeme", d.toString());
- }
- }
-
- @Test
- public void testPostRemove() throws Exception {
- setup(null);
- Result res = testFeed(xmlFilesPath + "test_removes", "remove?");
- assertEquals(2, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
- DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
- assertEquals("doc:test:remove1", d.toString());
- }
-
- {
- Message m = res.messages.get(1);
- assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
- DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
- assertEquals("doc:test:remove2", d.toString());
- }
- }
-
- @Test
- public void testRemoveBogusId() throws Exception {
- try {
- setup(null);
- Result res = testRequest(HttpRequest.createTestRequest("remove?id=unknowndoc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
- assertTrue(false);
- } catch (Exception e) {
- }
- }
-
- @Test
- public void testMultiRemove() throws Exception {
- setup(null);
- Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=doc:test:removeme&id[1]=doc:test:remove2&id[2]=doc:test:remove3", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
- assertEquals(3, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
- DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
- assertEquals("doc:test:removeme", d.toString());
- }
-
- {
- Message m = res.messages.get(1);
- assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
- DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
- assertEquals("doc:test:remove2", d.toString());
- }
-
- {
- Message m = res.messages.get(2);
- assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
- DocumentId d = ((RemoveDocumentMessage)m).getDocumentId();
- assertEquals("doc:test:remove3", d.toString());
- }
- }
-
- @Test
- public void testMultiRemoveSameDoc() throws Exception {
- setup(null);
- Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=userdoc:footype:1234:foo&id[1]=userdoc:footype:1234:foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT));
- assertEquals(2, res.messages.size());
-
- {
- Message m = res.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
- }
-
- {
- Message m = res.messages.get(1);
- assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType());
- }
- }
-
- @Test
- public void testFeedHandlerStatusCreation() throws Exception {
- VespaFeedHandlerStatus status = new VespaFeedHandlerStatus(
- new FeedContext(new MessagePropertyProcessor(
- new FeederConfig(new FeederConfig.Builder()),
- new LoadTypeConfig(new LoadTypeConfig.Builder())),
- factory, null, new ClusterList(), new NullFeedMetric(true)),
- true, true,
- Executors.newCachedThreadPool());
- }
-
- private class TestDocProc extends DocumentProcessor {
- @Override
- public Progress process(Processing processing) {
- for (DocumentOperation op : processing.getDocumentOperations()) {
- if (op instanceof DocumentPut) {
- Document document = ((DocumentPut)op).getDocument();
- document.setFieldValue("last_downloaded", new IntegerFieldValue(1234));
- }
- }
- return Progress.DONE;
- }
- }
-
- private class TestLaterDocProc extends DocumentProcessor {
- private final Logger log = Logger.getLogger(TestLaterDocProc.class.getName());
-
- private int counter = 0;
- @Override
- public Progress process(Processing processing) {
- synchronized (this) {
- counter++;
- if (counter % 2 == 1) {
- log.info("Returning LATER.");
- return Progress.LATER;
- }
- log.info("Returning DONE.");
- return Progress.DONE;
- }
- }
- }
-
- private Result testRequest(HttpRequest req) throws Exception {
- HttpResponse response = null;
- String feedPrefix = "feed";
- String removePrefix = "remove";
- String feedStatusPrefix = "feedstatus";
- String removeLocationPrefix = "removelocation";
-
- if (req.getUri().getPath().startsWith(feedPrefix)) {
- response = feedHandler.handle(req);
- }
- if (req.getUri().getPath().startsWith(removePrefix)) {
- response = removeHandler.handle(req);
- }
- if (req.getUri().getPath().startsWith(feedStatusPrefix)) {
- response = statusHandler.handle(req);
- }
- if (req.getUri().getPath().startsWith(removeLocationPrefix)) {
- response = removeLocationHandler.handle(req);
- }
-
- ByteArrayOutputStream output = new ByteArrayOutputStream();
- response.render(output);
-
- Result res = new Result();
- res.messages = factory.messages;
- res.output = new String(output.toByteArray());
-
- if (response instanceof FeedResponse) {
- FeedResponse feedResponse = (FeedResponse)response;
- res.error = feedResponse.getErrorMessageList().isEmpty() ? null : feedResponse.getErrorMessageList().get(0);
- res.errorCount = feedResponse.getErrorMessageList().size();
- assertTrue(feedResponse.isSuccess() == (res.errorCount == 0));
- }
- return res;
- }
-
- private Result testFeed(String xmlFile, String request) throws Exception {
- return testRequest(new FileRequest(new File(xmlFile), request).toRequest());
- }
-
- private Result testFeedGZIP(String xmlFile, String request) throws Exception {
- return testRequest(new FileRequest(new File(xmlFile), request, true).toRequest());
- }
-
- private class FileRequest {
-
- private final String req;
- private final File f;
- private boolean gzip = false;
-
- FileRequest(File f, String req) {
- this.req = req;
- this.f = f;
- }
-
- FileRequest(File f, String req, boolean gzip) {
- this.f = f;
- this.req = req;
- this.gzip = gzip;
- }
-
- public InputStream getData() {
- try {
- InputStream fileStream = new FileInputStream(f);
- if (gzip) {
- // Not exactly pretty, but in lack of an elegant way of transcoding
- ByteArrayOutputStream rawOut = new ByteArrayOutputStream();
- GZIPOutputStream compressed = new GZIPOutputStream(rawOut);
- byte[] buffer = new byte[1024];
- int read = -1;
- while (true) {
- read = fileStream.read(buffer);
- if (read == -1) break;
- compressed.write(buffer, 0, read);
- }
- compressed.finish();
- compressed.flush();
- rawOut.flush();
- return new ByteArrayInputStream(rawOut.toByteArray());
- }
- return fileStream;
- } catch (Exception e) {
- return null;
- }
- }
-
- public void addHeaders(HeaderFields headers) {
- headers.add("Content-Type", "image/jpeg");
- if (gzip)
- headers.add("Content-Encoding", "gzip");
- }
-
- public HttpRequest toRequest() {
- HttpRequest request = HttpRequest.createTestRequest(req, com.yahoo.jdisc.http.HttpRequest.Method.GET, getData());
- addHeaders(request.getJDiscRequest().headers());
- return request;
- }
-
- }
-
- private class Result {
- private List<Message> messages;
- private String output;
- private com.yahoo.processing.request.ErrorMessage error;
- private int errorCount;
- }
-
-}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java
deleted file mode 100644
index b7a911109cc..00000000000
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java
+++ /dev/null
@@ -1,104 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.storage.searcher;
-
-import com.yahoo.document.BucketId;
-import com.yahoo.documentapi.ProgressToken;
-import com.yahoo.documentapi.VisitorIterator;
-import org.junit.Test;
-
-import java.util.Set;
-import java.util.TreeSet;
-
-import static org.junit.Assert.*;
-
-@SuppressWarnings("deprecation")
-public class ContinuationHitTest {
-
- private static final String SINGLE_BUCKET_URL_SAFE_BASE64
- = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAHqNFZ4mrz-_wAAAAAAAAAA";
- private static final String MULTI_BUCKET_URL_SAFE_BASE64
- = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz--gAAAAAAAAAA6" +
- "jRWeJq8_vsAAAAAAAAAAOo0VniavP7_AAAAAAAAAAA=";
-
- @Test
- public void continuationTokensAreUrlSafeBase64Encoded() throws Exception {
- ContinuationHit hit = new ContinuationHit(createSingleBucketProgress());
- // We want -_ instead of +/
- assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, hit.getValue());
- }
-
- @Test
- public void continuationTokensAreNotBrokenIntoMultipleLines() throws Exception {
- ContinuationHit hit = new ContinuationHit(createMultiBucketProgress());
- assertTrue(hit.getValue().length() > 76); // Ensure we exceed MIME line length limits.
- assertFalse(hit.getValue().contains("\n"));
- }
-
- @Test
- public void decodingAcceptsUrlSafeTokens() throws Exception {
- final ProgressToken token = ContinuationHit.getToken(SINGLE_BUCKET_URL_SAFE_BASE64);
- // Roundtrip should yield identical results.
- assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64,
- new ContinuationHit(token).getValue());
- }
-
- /**
- * Legacy Base64 encoder emitted MIME Base64. Ensure we handle tokens from that era.
- */
- @Test
- public void decodingAcceptsLegacyNonUrlSafeTokens() throws Exception {
- final String legacyBase64 = convertedToMimeBase64Chars(SINGLE_BUCKET_URL_SAFE_BASE64);
- final ProgressToken legacyToken = ContinuationHit.getToken(legacyBase64);
-
- assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64,
- new ContinuationHit(legacyToken).getValue());
- }
-
- /**
- * Legacy Base64 encoder would happily output line breaks after each MIME line
- * boundary. Ensure we handle these gracefully.
- */
- @Test
- public void decodingAcceptsLegacyMimeLineBrokenTokens() throws Exception {
- final String multiBucketLegacyToken =
- "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz++gAAAAAAAAAA6jRWeJq8/vsA\r\n" +
- "AAAAAAAAAOo0VniavP7/AAAAAAAAAAA=";
- final ProgressToken legacyToken = ContinuationHit.getToken(multiBucketLegacyToken);
-
- assertEquals(MULTI_BUCKET_URL_SAFE_BASE64,
- new ContinuationHit(legacyToken).getValue());
- }
-
- /**
- * Returns a ProgressToken whose base 64 representation will be _less_ than 76 bytes (MIME line limit)
- */
- private ProgressToken createSingleBucketProgress() {
- ProgressToken token = new ProgressToken(16);
- // Use explicit bucket set so we can better control the binary representation
- // of the buckets, and thus the values written as base 64.
- Set<BucketId> buckets = new TreeSet<>();
- // This particular bucket ID will contain +/ chars when output as non-URL safe base 64.
- buckets.add(new BucketId(58, 0x123456789abcfeffL));
- VisitorIterator.createFromExplicitBucketSet(buckets, 16, token); // "Prime" the token.
- return token;
- }
-
- /**
- * Returns a ProgressToken whose base 64 representation will be _more_ than 76 bytes (MIME line limit)
- */
- private ProgressToken createMultiBucketProgress() {
- ProgressToken token = new ProgressToken(16);
- Set<BucketId> buckets = new TreeSet<>();
- buckets.add(new BucketId(58, 0x123456789abcfeffL));
- buckets.add(new BucketId(58, 0x123456789abcfefaL));
- buckets.add(new BucketId(58, 0x123456789abcfefbL));
- VisitorIterator.createFromExplicitBucketSet(buckets, 16, token); // "Prime" the token.
- return token;
- }
-
- private String convertedToMimeBase64Chars(String token) {
- // Doesn't split on MIME line boundaries, so not fully MIME compliant.
- return token.replace('-', '+').replace('_', '/');
- }
-
-}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java
deleted file mode 100755
index 2015efa4d5d..00000000000
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java
+++ /dev/null
@@ -1,129 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.storage.searcher;
-
-import com.yahoo.document.Document;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.DocumentType;
-import com.yahoo.documentapi.VisitorParameters;
-import com.yahoo.documentapi.VisitorSession;
-import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
-import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply;
-import com.yahoo.feedapi.DummySessionFactory;
-import com.yahoo.feedapi.SendSession;
-import com.yahoo.jdisc.Metric;
-import com.yahoo.messagebus.*;
-import com.yahoo.messagebus.Error;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.HashMap;
-
-/**
- * Used to automatically reply with a GetDocumentReply for every received GetDocumentMessage
- */
-public class DocumentSessionFactory extends DummySessionFactory {
-
- private DocumentType docType;
- private Error error;
- // Reply instances are shared between the two collections.
- List<GetDocumentReply> autoReplies;
- Map<DocumentId, GetDocumentReply> autoReplyLookup = new HashMap<>();
- boolean autoReply = true;
- boolean nullReply = false;
- private int sessionsCreated = 0;
-
- private class DocumentReplySession extends SendSession {
-
- ReplyHandler handler;
- Error e;
- DummySessionFactory owner;
-
- public DocumentReplySession(ReplyHandler handler, Error e, DummySessionFactory owner) {
- this.handler = handler;
- this.e = e;
- this.owner = owner;
- }
-
- protected Result onSend(Message m, boolean blockIfQueueFull) throws InterruptedException {
- if (!(m instanceof GetDocumentMessage)) {
- throw new IllegalArgumentException("Expected GetDocumentMessage");
- }
- GetDocumentMessage gm = (GetDocumentMessage)m;
- owner.messages.add(m);
- if (autoReply) {
- Document replyDoc;
- if (!nullReply) {
- replyDoc = new Document(docType, gm.getDocumentId());
- } else {
- replyDoc = null;
- }
- Reply r = new GetDocumentReply(replyDoc);
- r.setMessage(m);
- r.setContext(m.getContext());
- if (e != null) {
- r.addError(e);
- }
- handler.handleReply(r);
- } else if (owner.messages.size() == autoReplies.size()) {
- // Pair up all replies with their messages
- for (Message msg : owner.messages) {
- GetDocumentReply reply = autoReplyLookup.get(((GetDocumentMessage)msg).getDocumentId());
- reply.setMessage(msg);
- reply.setContext(msg.getContext());
- if (e != null) {
- reply.addError(e);
- }
- }
- // Now send them in the correct order. Instances are shared, so source
- // messages and contexts are properly set
- for (Reply reply : autoReplies) {
- handler.handleReply(reply);
- }
- }
-
- return Result.ACCEPTED;
- }
-
- public void close() {
- }
- }
-
- public DocumentSessionFactory(DocumentType docType) {
- this.docType = docType;
- this.error = null;
- }
-
- public DocumentSessionFactory(DocumentType docType, Error error, boolean autoReply, GetDocumentReply... autoReplies) {
- this.docType = docType;
- this.error = error;
- this.autoReplies = Arrays.asList(autoReplies);
- for (GetDocumentReply reply : autoReplies) {
- this.autoReplyLookup.put(reply.getDocument().getId(), reply);
- }
- this.autoReply = autoReply;
- }
-
- public boolean isNullReply() {
- return nullReply;
- }
-
- public void setNullReply(boolean nullReply) {
- this.nullReply = nullReply;
- }
-
- public int getSessionsCreated() {
- return sessionsCreated;
- }
-
- public SendSession createSendSession(ReplyHandler r, Metric metric) {
- ++sessionsCreated;
- return new DocumentReplySession(r, error, this);
- }
-
- @Override
- public VisitorSession createVisitorSession(VisitorParameters p) {
- return new DummyVisitorSession(p, docType);
- }
-
-}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java
deleted file mode 100644
index 02d26f8bd93..00000000000
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.storage.searcher;
-
-import com.yahoo.document.Document;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.DocumentPut;
-import com.yahoo.document.DocumentType;
-import com.yahoo.documentapi.*;
-import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
-import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;
-import com.yahoo.messagebus.Message;
-import com.yahoo.messagebus.Trace;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Stub to test visitors.
- */
-public class DummyVisitorSession implements VisitorSession {
-
- final VisitorParameters parameters;
- final DocumentType documentType;
- final List<Message> autoReplyMessages = new ArrayList<>();
-
- DummyVisitorSession(VisitorParameters p, DocumentType documentType) {
- parameters = p;
- this.documentType = documentType;
- p.getLocalDataHandler().setSession(this);
- addDefaultReplyMessages();
- }
-
- @Override
- public boolean isDone() {
- return true;
- }
-
- @Override
- public ProgressToken getProgress() {
- return new ProgressToken(12);
- }
-
- @Override
- public Trace getTrace() {
- return null;
- }
-
- public void addDocumentReply(String docId) {
- Document replyDoc = new Document(documentType, docId);
- autoReplyMessages.add(new PutDocumentMessage(new DocumentPut(replyDoc)));
- }
-
- public void addRemoveReply(String docId) {
- autoReplyMessages.add(new RemoveDocumentMessage(new DocumentId(docId)));
- }
-
- public void addDefaultReplyMessages() {
- addDocumentReply("userdoc:foo:1234:bar");
- if (parameters.visitRemoves()) {
- addRemoveReply("userdoc:foo:1234:removed");
- }
- }
-
- public void clearAutoReplyMessages() {
- autoReplyMessages.clear();
- }
-
- @Override
- public boolean waitUntilDone(long l) throws InterruptedException {
- for (Message msg : autoReplyMessages) {
- parameters.getLocalDataHandler().onMessage(msg, new AckToken(this));
- }
- return true;
- }
-
- @Override
- public void ack(AckToken ackToken) {
- }
-
- @Override
- public void abort() {
- }
-
- @Override
- public VisitorResponse getNext() {
- return null;
- }
-
- @Override
- public VisitorResponse getNext(int i) throws InterruptedException {
- return null;
- }
-
- @Override
- public void destroy() {
- }
-
-}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java
deleted file mode 100755
index 14fe0dc857d..00000000000
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java
+++ /dev/null
@@ -1,1093 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.storage.searcher;
-
-import com.yahoo.component.chain.Chain;
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.feedhandler.NullFeedMetric;
-import com.yahoo.jdisc.HeaderFields;
-import com.yahoo.vespa.config.content.LoadTypeConfig;
-import com.yahoo.document.*;
-import com.yahoo.document.datatypes.IntegerFieldValue;
-import com.yahoo.document.datatypes.Raw;
-import com.yahoo.document.datatypes.StringFieldValue;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
-import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
-import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply;
-import com.yahoo.feedapi.FeedContext;
-import com.yahoo.feedapi.MessagePropertyProcessor;
-import com.yahoo.messagebus.Message;
-import com.yahoo.messagebus.routing.Route;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.Searcher;
-import com.yahoo.search.result.Hit;
-import com.yahoo.search.result.HitGroup;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.vespaclient.ClusterList;
-import com.yahoo.vespaclient.config.FeederConfig;
-
-import org.junit.Test;
-
-import java.io.*;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.zip.GZIPOutputStream;
-
-import static org.junit.Assert.*;
-
-@SuppressWarnings("deprecation")
-public class GetSearcherTestCase {
-
- private DocumentTypeManager docMan = null;
- private DocumentType docType;
- private FeederConfig defFeedCfg = new FeederConfig(new FeederConfig.Builder());
- private LoadTypeConfig defLoadTypeCfg = new LoadTypeConfig(new LoadTypeConfig.Builder());
-
- @org.junit.Before
- public void setUp() {
- docMan = new DocumentTypeManager();
- docType = new DocumentType("kittens");
- docType.addHeaderField("name", DataType.STRING);
- docType.addField("description", DataType.STRING);
- docType.addField("image", DataType.STRING);
- docType.addField("fluffiness", DataType.INT);
- docType.addField("foo", DataType.RAW);
- docMan.registerDocumentType(docType);
- }
-
- @org.junit.After
- public void tearDown() {
- docMan = null;
- docType = null;
- }
-
- private void assertHits(HitGroup hits, String... wantedHits) {
- assertEquals(wantedHits.length, hits.size());
- for (int i = 0; i < wantedHits.length; ++i) {
- assertTrue(hits.get(i) instanceof DocumentHit);
- DocumentHit hit = (DocumentHit)hits.get(i);
- assertEquals(wantedHits[i], hit.getDocument().getId().toString());
- }
- }
-
- @Test
- public void testGetSingleDocumentQuery() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:1:2"));
- System.out.println("HTTP request is " + result.getQuery().getHttpRequest());
-
- assertEquals(1, factory.messages.size());
- {
- Message m = factory.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
- GetDocumentMessage gdm = (GetDocumentMessage)m;
- DocumentId d = gdm.getDocumentId();
- assertEquals("userdoc:kittens:1:2", d.toString());
- assertEquals("[all]", gdm.getFieldSet());
- }
- assertEquals(1, result.hits().size());
- assertHits(result.hits(), "userdoc:kittens:1:2");
- // By default, document hit should not have its hit fields set
- DocumentHit hit = (DocumentHit)result.hits().get(0);
- assertEquals(0, hit.fieldKeys().size());
- }
-
- @Test
- public void testGetMultipleDocumentsQuery() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- Query query = newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4");
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query);
-
- assertEquals(2, factory.messages.size());
- {
- Message m = factory.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
- GetDocumentMessage gdm = (GetDocumentMessage)m;
- DocumentId d = gdm.getDocumentId();
- assertEquals("userdoc:kittens:1:2", d.toString());
- assertEquals("[all]", gdm.getFieldSet());
- }
-
- {
- Message m = factory.messages.get(1);
- assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
- GetDocumentMessage gdm = (GetDocumentMessage)m;
- DocumentId d = gdm.getDocumentId();
- assertEquals("userdoc:kittens:3:4", d.toString());
- assertEquals("[all]", gdm.getFieldSet());
- }
- assertEquals(2, result.hits().size());
- assertNull(result.hits().getErrorHit());
- assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4");
- assertEquals(2, query.getHits());
- }
-
- // Test that you can use both query string and POSTed IDs
- @Test
- public void testGetMultipleDocumentsQueryAndPOST() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- String data = "userdoc:kittens:5:6\nuserdoc:kittens:7:8\nuserdoc:kittens:9:10";
- MockHttpRequest request = new MockHttpRequest(data.getBytes("utf-8"), "/get/?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4");
- Query query = new Query(request.toRequest());
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query);
-
- assertEquals(5, factory.messages.size());
- assertEquals(5, result.hits().size());
- assertNull(result.hits().getErrorHit());
- assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4",
- "userdoc:kittens:5:6", "userdoc:kittens:7:8", "userdoc:kittens:9:10");
- }
-
- @Test
- public void testGetMultipleDocumentsQueryAndGZippedPOST() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- String data = "userdoc:kittens:5:6\nuserdoc:kittens:7:8\nuserdoc:kittens:9:10";
-
- // Create with automatic GZIP encoding
- MockHttpRequest request = new MockHttpRequest(data.getBytes("utf-8"), "/get/?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4", true);
- Query query = new Query(request.toRequest());
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query);
-
- assertEquals(5, factory.messages.size());
- assertEquals(5, result.hits().size());
- assertNull(result.hits().getErrorHit());
- assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4",
- "userdoc:kittens:5:6", "userdoc:kittens:7:8", "userdoc:kittens:9:10");
- }
-
- /* Test that a query without any ids is passed through to the next chain */
- @Test
- public void testQueryPassThrough() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- HitGroup hits = new HitGroup("mock");
- hits.add(new Hit("blernsball"));
- Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(hits));
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?flarn=blern"));
-
- assertEquals(0, factory.messages.size());
- assertEquals(1, result.hits().size());
- assertNotNull(result.hits().get("blernsball"));
- }
-
- /* Test that a query will contain both document hits and hits from a searcher
- * further down the chain, iff the searcher returns a DocumentHit.
- */
- @Test
- public void testQueryPassThroughAndGet() throws Exception {
- Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:1234:foo"));
- doc1.setFieldValue("name", new StringFieldValue("megacat"));
- doc1.setFieldValue("description", new StringFieldValue("supercat"));
- doc1.setFieldValue("fluffiness", new IntegerFieldValue(10000));
- GetDocumentReply[] replies = new GetDocumentReply[] {
- new GetDocumentReply(doc1)
- };
-
- DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- DocumentHit backendHit = new DocumentHit(new Document(docType, new DocumentId("userdoc:kittens:5678:bar")), 5);
- Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(backendHit));
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?query=flarn&id=userdoc:kittens:1234:foo"));
-
- assertEquals(1, factory.messages.size());
- assertEquals(2, result.hits().size());
- assertNotNull(result.hits().get("userdoc:kittens:5678:bar"));
- assertNotNull(result.hits().get("userdoc:kittens:1234:foo"));
- }
-
- @Test
- public void testQueryPassThroughAndGetUnknownBackendHit() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- HitGroup hits = new HitGroup("mock");
- hits.add(new Hit("blernsball"));
- Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(hits));
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?flarn=blern&id=userdoc:kittens:9:aaa"));
-
- assertEquals(0, factory.messages.size());
- assertNotNull(result.hits().getErrorHit());
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"searcher\" code=\"18\" message=\"Internal server error.: " +
- "A backend searcher to com.yahoo.storage.searcher.GetSearcher returned a " +
- "hit that was not an instance of com.yahoo.storage.searcher.DocumentHit. " +
- "Only DocumentHit instances are supported in the backend hit result set " +
- "when doing queries that contain document identifier sets recognised by the " +
- "Get Searcher.\"/>\n" +
- "</errors>\n" +
- "</result>\n", result);
- }
-
- @Test
- public void testConfig() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(58).route("route66").retryenabled(false)), defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=doc:batman:dahnahnahnah"));
-
- assertEquals(1, factory.messages.size());
- {
- Message m = factory.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
- GetDocumentMessage gdm = (GetDocumentMessage)m;
- DocumentId d = gdm.getDocumentId();
- assertEquals("doc:batman:dahnahnahnah", d.toString());
- assertEquals("[all]", gdm.getFieldSet());
- assertEquals(Route.parse("route66"), gdm.getRoute());
- assertFalse(gdm.getRetryEnabled());
- assertTrue(58000 >= gdm.getTimeRemaining());
- }
- }
-
- @Test
- public void testConfigChanges() throws Exception {
- String config = "raw:timeout 37\nroute \"riksveg18\"\nretryenabled true";
- DocumentSessionFactory factory = new DocumentSessionFactory(docType);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(58).route("riksveg18").retryenabled(true)),
- defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=doc:batman:dahnahnahnah"));
-
- assertEquals(1, factory.messages.size());
- assertEquals(1, factory.getSessionsCreated());
- {
- Message m = factory.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
- GetDocumentMessage gdm = (GetDocumentMessage)m;
- DocumentId d = gdm.getDocumentId();
- assertEquals("doc:batman:dahnahnahnah", d.toString());
- assertEquals("[all]", gdm.getFieldSet());
- assertEquals(Route.parse("riksveg18"), gdm.getRoute());
- assertTrue(gdm.getRetryEnabled());
- assertTrue(58000 >= gdm.getTimeRemaining());
- }
-
- factory.messages.clear();
-
- FeederConfig newConfig = new FeederConfig(new FeederConfig.Builder()
- .timeout(123)
- .route("e6")
- .retryenabled(false)
- );
- searcher.getMessagePropertyProcessor().configure(newConfig, defLoadTypeCfg);
-
- new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id=doc:spiderman:does_whatever_a_spider_can"));
-
- // riksveg18 is created again, and e6 is created as well.
- assertEquals(3, factory.getSessionsCreated());
-
- assertEquals(1, factory.messages.size());
- {
- Message m = factory.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
- GetDocumentMessage gdm = (GetDocumentMessage)m;
- DocumentId d = gdm.getDocumentId();
- assertEquals("doc:spiderman:does_whatever_a_spider_can", d.toString());
- assertEquals("[all]", gdm.getFieldSet());
- assertEquals(Route.parse("e6"), gdm.getRoute());
- assertFalse(gdm.getRetryEnabled());
- assertTrue(123000 >= gdm.getTimeRemaining());
- }
- }
-
- @Test
- public void testQueryOverridesDefaults() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&priority=LOW_2&route=highwaytohell&timeout=58"));
-
- assertEquals(2, factory.messages.size());
- long lastTimeout = 58000;
- {
- Message m = factory.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
- GetDocumentMessage gdm = (GetDocumentMessage)m;
- DocumentId d = gdm.getDocumentId();
- assertEquals("userdoc:kittens:1:2", d.toString());
- assertEquals("[all]", gdm.getFieldSet());
- assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority());
- assertEquals(Route.parse("highwaytohell"), gdm.getRoute());
- assertTrue(lastTimeout >= gdm.getTimeRemaining());
- lastTimeout = gdm.getTimeRemaining();
- }
-
- {
- Message m = factory.messages.get(1);
- assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
- GetDocumentMessage gdm = (GetDocumentMessage)m;
- DocumentId d = gdm.getDocumentId();
- assertEquals("userdoc:kittens:3:4", d.toString());
- assertEquals("[all]", gdm.getFieldSet());
- assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority());
- assertEquals(Route.parse("highwaytohell"), gdm.getRoute());
- assertTrue(lastTimeout >= gdm.getTimeRemaining());
- }
- }
-
- @Test
- public void testQueryOverridesConfig() throws Exception {
- String config = "raw:timeout 458\nroute \"route66\"";
- DocumentSessionFactory factory = new DocumentSessionFactory(docType);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&priority=LOW_2&route=highwaytohell&timeout=123"));
-
- long lastTimeout = 123000;
- assertEquals(2, factory.messages.size());
- {
- Message m = factory.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
- GetDocumentMessage gdm = (GetDocumentMessage)m;
- DocumentId d = gdm.getDocumentId();
- assertEquals("userdoc:kittens:1:2", d.toString());
- assertEquals("[all]", gdm.getFieldSet());
- assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority());
- assertEquals(Route.parse("highwaytohell"), gdm.getRoute());
- assertTrue(lastTimeout >= gdm.getTimeRemaining());
- lastTimeout = gdm.getTimeRemaining();
- }
-
- {
- Message m = factory.messages.get(1);
- assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
- GetDocumentMessage gdm = (GetDocumentMessage)m;
- DocumentId d = gdm.getDocumentId();
- assertEquals("userdoc:kittens:3:4", d.toString());
- assertEquals("[all]", gdm.getFieldSet());
- assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority());
- assertEquals(Route.parse("highwaytohell"), gdm.getRoute());
- assertTrue(lastTimeout >= gdm.getTimeRemaining());
- }
- }
-
- @Test
- public void testBadPriorityValue() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id=userdoc:kittens:1:2&priority=onkel_jubalon"));
-
- assertNotNull(result.hits().getErrorHit());
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " +
- "java.lang.IllegalArgumentException: No enum const" +
- "ant " +
- "com.yahoo.documentapi.messagebus.protocol.DocumentProtocol" +
- "." +
- "Priority.onkel_jubalon\"/>\n" +
- "</errors>\n" +
- "</result>\n", result);
- }
-
- @Test
- public void testMultiIdBadArrayIndex() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- {
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id[1]=userdoc:kittens:1:2"));
-
- assertNotNull(result.hits().getErrorHit());
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " +
- "java.lang.IllegalArgumentException: query contains document ID " +
- "array that is not zero-based and/or linearly increasing\"/>\n" +
- "</errors>\n" +
- "</result>\n", result);
- }
-
- {
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id[0]=userdoc:kittens:1:2&id[2]=userdoc:kittens:2:3"));
-
- assertNotNull(result.hits().getErrorHit());
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " +
- "java.lang.IllegalArgumentException: query contains document ID " +
- "array that is not zero-based and/or linearly increasing\"/>\n" +
- "</errors>\n" +
- "</result>\n", result);
- }
-
- {
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id[1]=userdoc:kittens:2:3"));
-
- assertNotNull(result.hits().getErrorHit());
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " +
- "java.lang.IllegalArgumentException: query contains document ID " +
- "array that is not zero-based and/or linearly increasing\"/>\n" +
- "</errors>\n" +
- "</result>\n", result);
- }
-
- {
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id[0=userdoc:kittens:1:2"));
-
- assertNotNull(result.hits().getErrorHit());
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " +
- "java.lang.IllegalArgumentException: Malformed document ID array parameter\"/>\n" +
- "</errors>\n" +
- "</result>\n", result);
- }
- }
-
- @Test
- public void testLegacyHeadersOnly() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id=userdoc:kittens:1:2&headersonly=true"));
-
- assertEquals(1, factory.messages.size());
- {
- Message m = factory.messages.get(0);
- assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType());
- GetDocumentMessage gdm = (GetDocumentMessage)m;
- DocumentId d = gdm.getDocumentId();
- assertEquals("userdoc:kittens:1:2", d.toString());
- assertEquals("[header]", gdm.getFieldSet());
- }
- assertEquals(1, result.hits().size());
- assertHits(result.hits(), "userdoc:kittens:1:2");
- }
-
- @Test
- public void testFieldSet() throws Exception {
- }
-
- @Test
- public void testConsistentResultOrdering() throws Exception {
- GetDocumentReply[] replies = new GetDocumentReply[] {
- new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:1:2"))),
- new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:7:8"))),
- new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:555:123")))
- };
-
- // Use a predefined reply list to ensure messages are answered out of order
- DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id[0]=userdoc:kittens:555:123&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:7:8"));
-
- assertEquals(3, factory.messages.size());
- assertEquals(3, result.hits().size());
- // Hits must be in the same order as their document IDs in the query
- assertHits(result.hits(), "userdoc:kittens:555:123", "userdoc:kittens:1:2", "userdoc:kittens:7:8");
-
- assertEquals(0, ((DocumentHit)result.hits().get(0)).getIndex());
- assertEquals(1, ((DocumentHit)result.hits().get(1)).getIndex());
- assertEquals(2, ((DocumentHit)result.hits().get(2)).getIndex());
- }
-
- @Test
- public void testResultWithSingleError() throws Exception {
- com.yahoo.messagebus.Error err = new com.yahoo.messagebus.Error(32, "Alas, it went poorly");
- DocumentSessionFactory factory = new DocumentSessionFactory(docType, err, true);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4"));
- assertNotNull(result.hits().getErrorHit());
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"messagebus\" code=\"32\" message=\"Alas, it went poorly\"/>\n" +
- "</errors>\n" +
- "</result>\n", result);
- }
-
- @Test
- public void testResultWithMultipleErrors() throws Exception {
- Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:77:88"));
- Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:99:111"));
- GetDocumentReply errorReply1 = new GetDocumentReply(doc1);
- errorReply1.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 had fleas."));
- GetDocumentReply errorReply2 = new GetDocumentReply(doc2);
- errorReply2.addError(new com.yahoo.messagebus.Error(456, "userdoc:kittens:99:111 shredded the curtains."));
- GetDocumentReply[] replies = new GetDocumentReply[] {
- errorReply1,
- errorReply2
- };
-
- Chain<Searcher> searchChain = createSearcherChain(replies);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:99:111"));
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 had fleas.\"/>\n" +
- "<error type=\"messagebus\" code=\"456\" message=\"userdoc:kittens:99:111 shredded the curtains.\"/>\n" +
- "</errors>\n" +
- "</result>\n", result);
- }
-
- @Test
- public void testResultWithNullDocument() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, true);
- factory.setNullReply(true);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id[0]=userdoc:kittens:55:bad_document_id"));
- // Document not found does not produce any hit at all, error or otherwise
- assertNull(result.hits().getErrorHit());
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "</result>\n", result);
- }
-
- @Test
- public void testDefaultDocumentHitRendering() throws Exception {
- Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:3:4"));
- doc1.setFieldValue("name", new StringFieldValue("mittens"));
- doc1.setFieldValue("description", new StringFieldValue("it's a cat"));
- doc1.setFieldValue("fluffiness", new IntegerFieldValue(8));
- Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:1:2"));
- doc2.setFieldValue("name", new StringFieldValue("garfield"));
- doc2.setFieldValue("description",
- new StringFieldValue("preliminary research indicates <em>hatred</em> of mondays. caution advised"));
- doc2.setFieldValue("fluffiness", new IntegerFieldValue(2));
- Document doc3 = new Document(docType, new DocumentId("userdoc:kittens:77:88"));
- GetDocumentReply errorReply = new GetDocumentReply(doc3);
- errorReply.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 had fleas."));
- GetDocumentReply[] replies = new GetDocumentReply[] {
- new GetDocumentReply(doc1),
- new GetDocumentReply(doc2),
- errorReply
- };
-
- // Use a predefined reply list to ensure messages are answered out of order
- Chain<Searcher> searchChain = createSearcherChain(replies);
-
- Result xmlResult = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:3:4"));
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 had fleas.\"/>\n" +
- "</errors>\n" +
- "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:1:2\">\n" +
- " <name>garfield</name>\n" +
- " <description>preliminary research indicates &lt;em&gt;hatred&lt;/em&gt; of mondays. caution advised</description>\n" +
- " <fluffiness>2</fluffiness>\n" +
- "</document>\n" +
- "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:3:4\">\n" +
- " <name>mittens</name>\n" +
- " <description>it's a cat</description>\n" +
- " <fluffiness>8</fluffiness>\n" +
- "</document>\n" +
- "</result>\n", xmlResult);
- }
-
- @Test
- public void testDocumentFieldNoContentType() throws Exception {
- Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1"));
- doc1.setFieldValue("name", "derrick");
- doc1.setFieldValue("description", "kommisar katze");
- doc1.setFieldValue("fluffiness", 0);
- GetDocumentReply[] replies = new GetDocumentReply[] {
- new GetDocumentReply(doc1),
- };
- Chain<Searcher> searchChain = createSearcherChain(replies);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id=userdoc:kittens:5:1&field=description"));
-
- assertNull(result.hits().getErrorHit());
- assertEquals("text/xml", result.getTemplating().getTemplates().getMimeType());
- assertEquals("UTF-8", result.getTemplating().getTemplates().getEncoding());
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>kommisar katze</result>\n", result);
- }
-
- @Test
- public void testDocumentFieldEscapeXML() throws Exception {
- Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1"));
- doc1.setFieldValue("name", "asfd");
- doc1.setFieldValue("description", "<script type=\"evil/madness\">horror & screams</script>");
- doc1.setFieldValue("fluffiness", 0);
- GetDocumentReply[] replies = new GetDocumentReply[] {
- new GetDocumentReply(doc1),
- };
- Chain<Searcher> searchChain = createSearcherChain(replies);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id=userdoc:kittens:5:1&field=description"));
-
- assertNull(result.hits().getErrorHit());
- assertEquals("text/xml", result.getTemplating().getTemplates().getMimeType());
- assertEquals("UTF-8", result.getTemplating().getTemplates().getEncoding());
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>&lt;script type=\"evil/madness\"&gt;horror &amp; screams&lt;/script&gt;</result>\n", result);
- }
-
- @Test
- public void testDocumentFieldRawContent() throws Exception {
- byte[] contentBytes = new byte[] { 0, -128, 127 };
-
- Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456"));
- doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes)));
- GetDocumentReply[] replies = new GetDocumentReply[] {
- new GetDocumentReply(doc1)
- };
-
- Chain<Searcher> searchChain = createSearcherChain(replies);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id=userdoc:kittens:123:456&field=foo"));
-
- assertNull(result.hits().getErrorHit());
- assertEquals("application/octet-stream", result.getTemplating().getTemplates().getMimeType());
-
- ByteArrayOutputStream stream = new ByteArrayOutputStream();
- com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result);
- stream.flush();
-
- byte[] resultBytes = stream.toByteArray();
- assertEquals(contentBytes.length, resultBytes.length);
- for (int i = 0; i < resultBytes.length; ++i) {
- assertEquals(contentBytes[i], resultBytes[i]);
- }
- }
-
- @Test
- public void testDocumentFieldRawWithContentOverride() throws Exception {
- byte[] contentBytes = new byte[] { 0, -128, 127 };
-
- Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456"));
- doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes)));
- GetDocumentReply[] replies = new GetDocumentReply[] {
- new GetDocumentReply(doc1)
- };
-
- Chain<Searcher> searchChain = createSearcherChain(replies);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id=userdoc:kittens:123:456&field=foo&contenttype=text/fancy"));
-
- assertNull(result.hits().getErrorHit());
- assertEquals("text/fancy", result.getTemplating().getTemplates().getMimeType());
-
- ByteArrayOutputStream stream = new ByteArrayOutputStream();
- com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result);
- stream.flush();
-
- byte[] resultBytes = stream.toByteArray();
- assertEquals(contentBytes.length, resultBytes.length);
- for (int i = 0; i < resultBytes.length; ++i) {
- assertEquals(contentBytes[i], resultBytes[i]);
- }
- }
-
- @Test
- public void testDocumentFieldWithMultipleIDs() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&field=name"));
- assertNotNull(result.hits().getErrorHit());
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " +
- "java.lang.IllegalArgumentException: Field only valid for single document id query\"/>\n" +
- "</errors>\n" +
- "</result>\n", result);
- }
-
- @Test
- public void testDocumentFieldNotSet() throws Exception {
- Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1"));
- doc1.setFieldValue("name", "asdf");
- doc1.setFieldValue("description", "fdsafsdf");
- doc1.setFieldValue("fluffiness", 10);
- GetDocumentReply[] replies = new GetDocumentReply[] {
- new GetDocumentReply(doc1),
- };
- Chain<Searcher> searchChain = createSearcherChain(replies);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id=userdoc:kittens:5:1&field=image"));
-
- assertNotNull(result.hits().getErrorHit());
- assertEquals(1, result.hits().size());
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"searcher\" code=\"16\" message=\"Resource not found.: " +
- "Field 'image' found in document type, but had no content in userdoc:kittens:5:1\"/>\n" +
- "</errors>\n" +
- "</result>\n", result);
- }
-
-
- @Test
- public void testDocumentFieldWithDocumentNotFound() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, true);
- factory.setNullReply(true);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id=userdoc:kittens:1:2&field=name"));
- assertNotNull(result.hits().getErrorHit());
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"searcher\" code=\"16\" message=\"Resource not found.: " +
- "Document not found, could not return field 'name'\"/>\n" +
- "</errors>\n" +
- "</result>\n", result);
- }
-
- @Test
- public void testDocumentFieldNotReachableWithHeadersOnly() throws Exception {
- Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1"));
- doc1.setFieldValue("name", "asdf");
- // don't set body fields
- GetDocumentReply[] replies = new GetDocumentReply[] {
- new GetDocumentReply(doc1),
- };
- Chain<Searcher> searchChain = createSearcherChain(replies);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id=userdoc:kittens:5:1&field=description&headersonly=true"));
-
- assertNotNull(result.hits().getErrorHit());
- assertEquals(1, result.hits().size());
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"searcher\" code=\"4\" message=\"Invalid query parameter: " +
- "Field 'description' is located in document body, but headersonly " +
- "prevents it from being retrieved in userdoc:kittens:5:1\"/>\n" +
- "</errors>\n" +
- "</result>\n", result);
- }
-
- @Test
- public void testVespaXMLTemplate() throws Exception {
- Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:3:4"));
- doc1.setFieldValue("name", "mittens");
- doc1.setFieldValue("description", "it's a cat");
- doc1.setFieldValue("fluffiness", 8);
- Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:1:2"));
- doc2.setFieldValue("name", "garfield");
- doc2.setFieldValue("description", "preliminary research indicates <em>hatred</em> of mondays. caution advised");
- doc2.setFieldValue("fluffiness", 2);
- Document doc3 = new Document(docType, new DocumentId("userdoc:kittens:77:88"));
- GetDocumentReply errorReply = new GetDocumentReply(doc3);
- errorReply.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 lost in a <ni!>\"shrubbery\"</ni!>"));
- GetDocumentReply[] replies = new GetDocumentReply[] {
- new GetDocumentReply(doc1),
- new GetDocumentReply(doc2),
- errorReply
- };
-
- // Use a predefined reply list to ensure messages are answered out of order
- Chain<Searcher> searchChain = createSearcherChain(replies);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:3:4")); // TODO!
-
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 lost in a &lt;ni!&gt;&quot;shrubbery&quot;&lt;/ni!&gt;\"/>\n"+
- "</errors>\n" +
- "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:1:2\">\n" +
- " <name>garfield</name>\n" +
- " <description>preliminary research indicates &lt;em&gt;hatred&lt;/em&gt; of mondays. caution advised</description>\n" +
- " <fluffiness>2</fluffiness>\n" +
- "</document>\n" +
- "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:3:4\">\n" +
- " <name>mittens</name>\n" +
- " <description>it's a cat</description>\n" +
- " <fluffiness>8</fluffiness>\n" +
- "</document>\n" +
- "</result>\n", result);
- }
-
- @Test
- public void testDocumentHitWithPopulatedHitFields() throws Exception {
- Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:1234:foo"));
- doc1.setFieldValue("name", new StringFieldValue("megacat"));
- doc1.setFieldValue("description", new StringFieldValue("supercat"));
- doc1.setFieldValue("fluffiness", new IntegerFieldValue(10000));
- GetDocumentReply[] replies = new GetDocumentReply[] {
- new GetDocumentReply(doc1)
- };
-
- // Use a predefined reply list to ensure messages are answered out of order
- Chain<Searcher> searchChain = createSearcherChain(replies);
-
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("?id=userdoc:kittens:1234:foo&populatehitfields=true"));
- assertEquals(1, result.hits().size());
- assertHits(result.hits(), "userdoc:kittens:1234:foo");
-
- DocumentHit hit = (DocumentHit)result.hits().get(0);
- Iterator<Map.Entry<String, Object>> iter = hit.fieldIterator();
- Set<String> fieldSet = new TreeSet<>();
- while (iter.hasNext()) {
- Map.Entry<String, Object> kv = iter.next();
- StringBuilder field = new StringBuilder();
- field.append(kv.getKey()).append(" -> ").append(kv.getValue());
- fieldSet.add(field.toString());
- }
- StringBuilder fields = new StringBuilder();
- for (String s : fieldSet) {
- fields.append(s).append("\n");
- }
- assertEquals(
- "description -> supercat\n" +
- "documentid -> userdoc:kittens:1234:foo\n" +
- "fluffiness -> 10000\n" +
- "name -> megacat\n",
- fields.toString());
- }
-
- @Test
- public void deserializationExceptionsAreHandledGracefully() throws Exception {
- Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1"));
- GetDocumentReply[] replies = new GetDocumentReply[] {
- new MockFailingGetDocumentReply(doc1),
- };
- Chain<Searcher> searchChain = createSearcherChain(replies);
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:5:1"));
- assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<errors>\n" +
- "<error type=\"searcher\" code=\"18\" message=\"Internal server error.: " +
- "Got exception of type java.lang.RuntimeException during document " +
- "deserialization: epic dragon attack\"/>\n"+
- "</errors>\n" +
- "</result>\n", result);
- }
-
- @Test
- public void testJsonRendererSetting() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- Chain<Searcher> searchChain = new Chain<>(searcher);
-
- Query query = newQuery("?id=userdoc:kittens:1:2&format=json");
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query);
- assertFalse(result.getTemplating().getTemplates() instanceof DocumentXMLTemplate);
- }
-
- private Query newQuery(String queryString) {
- return new Query(HttpRequest.createTestRequest(queryString, com.yahoo.jdisc.http.HttpRequest.Method.GET));
- }
-
- private Chain<Searcher> createSearcherChain(GetDocumentReply[] replies) throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies);
- GetSearcher searcher = new GetSearcher(new FeedContext(
- new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg),
- factory, docMan, new ClusterList(), new NullFeedMetric(true)));
- return new Chain<>(searcher);
- }
-
- private static class MockFailingGetDocumentReply extends GetDocumentReply {
- private int countdown = 2;
-
- private MockFailingGetDocumentReply(Document doc) {
- super(doc);
- }
-
- @Override
- public Document getDocument() {
- // Reason for countdown is that the test DocumentSessionFactory calls
- // getDocument once internally before code can ever reach handleReply.
- if (--countdown == 0) {
- throw new RuntimeException("epic dragon attack");
- }
- return super.getDocument();
- }
- }
-
- private static class MockBackend extends Searcher {
- private Hit hitToReturn;
-
- public MockBackend(Hit hitToReturn) {
- this.hitToReturn = hitToReturn;
- }
-
- @Override
- public Result search(Query query, Execution execution) {
- Result result = new Result(query);
- result.hits().add(hitToReturn);
- return result;
- }
- }
-
- private class MockHttpRequest {
-
- private final String req;
- private byte[] data;
- private boolean gzip = false;
-
- MockHttpRequest(byte[] data, String req) {
- this.req = req;
- this.data = data;
- }
-
- MockHttpRequest(byte[] data, String req, boolean gzip) {
- this.data = data;
- this.req = req;
- this.gzip = gzip;
- }
-
- public InputStream getData() {
- if (gzip) {
- try {
- ByteArrayOutputStream rawOut = new ByteArrayOutputStream();
- GZIPOutputStream compressed = new GZIPOutputStream(rawOut);
- compressed.write(data, 0, data.length);
- compressed.finish();
- compressed.flush();
- rawOut.flush();
- return new ByteArrayInputStream(rawOut.toByteArray());
- } catch (Exception e) {
- return null;
- }
- }
- return new ByteArrayInputStream(data);
- }
-
- public void addHeaders(HeaderFields headers) {
- headers.add("Content-Type", "text/plain;encoding=UTF-8");
- if (gzip)
- headers.add("Content-Encoding", "gzip");
- }
-
- public com.yahoo.container.jdisc.HttpRequest toRequest() {
- com.yahoo.container.jdisc.HttpRequest request = com.yahoo.container.jdisc.HttpRequest.createTestRequest(req, com.yahoo.jdisc.http.HttpRequest.Method.GET, getData());
- addHeaders(request.getJDiscRequest().headers());
- return request;
- }
-
- }
-
- public static void assertRendered(String expected,Result result) throws Exception {
- assertRendered(expected,result,true);
- }
-
- public static void assertRendered(String expected,Result result,boolean checkFullEquality) throws Exception {
- if (checkFullEquality)
- assertEquals(expected, ResultRenderingUtil.getRendered(result));
- else
- assertTrue(ResultRenderingUtil.getRendered(result).startsWith(expected));
- }
-
-}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java
deleted file mode 100644
index 7f392b0565b..00000000000
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.storage.searcher;
-
-import com.yahoo.search.Result;
-
-import java.io.ByteArrayOutputStream;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-
-@SuppressWarnings("deprecation")
-public class ResultRenderingUtil {
-
- public static String getRendered(Result result) throws Exception {
- ByteArrayOutputStream stream = new ByteArrayOutputStream();
- Charset cs = Charset.forName("utf-8");
- CharsetDecoder decoder = cs.newDecoder();
- com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result);
- stream.flush();
- return decoder.decode(ByteBuffer.wrap(stream.toByteArray())).toString();
- }
-
-}
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java
deleted file mode 100644
index e49d2ca4db3..00000000000
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java
+++ /dev/null
@@ -1,243 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.storage.searcher;
-
-import com.yahoo.component.chain.Chain;
-import com.yahoo.cloud.config.ClusterListConfig;
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.protect.Error;
-import com.yahoo.documentapi.VisitorSession;
-import com.yahoo.feedhandler.NullFeedMetric;
-import com.yahoo.vespa.config.content.LoadTypeConfig;
-import com.yahoo.document.DataType;
-import com.yahoo.document.DocumentType;
-import com.yahoo.document.DocumentTypeManager;
-import com.yahoo.documentapi.VisitorParameters;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
-import com.yahoo.feedapi.FeedContext;
-import com.yahoo.feedapi.MessagePropertyProcessor;
-import com.yahoo.messagebus.StaticThrottlePolicy;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.Searcher;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.vdslib.VisitorOrdering;
-import com.yahoo.vespaclient.ClusterList;
-import com.yahoo.vespaclient.config.FeederConfig;
-
-import org.junit.Test;
-
-import java.util.Arrays;
-
-import static org.junit.Assert.*;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-@SuppressWarnings("deprecation")
-public class VisitorSearcherTestCase {
-
- private DocumentTypeManager docMan = null;
- private DocumentType docType;
- DocumentSessionFactory factory;
-
- @org.junit.Before
- public void setUp() {
- docMan = new DocumentTypeManager();
- docType = new DocumentType("kittens");
- docType.addHeaderField("name", DataType.STRING);
- docType.addField("description", DataType.STRING);
- docType.addField("image", DataType.RAW);
- docType.addField("fluffiness", DataType.INT);
- docType.addField("foo", DataType.RAW);
- docMan.registerDocumentType(docType);
- factory = new DocumentSessionFactory(docType);
- }
-
- public VisitSearcher create() throws Exception {
- ClusterListConfig.Storage.Builder storageCluster = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foobar").name("foobar");
- ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(storageCluster));
- ClusterList clusterList = new ClusterList(clusterListCfg);
- return new VisitSearcher(new FeedContext(
- new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(458).route("riksveg18").retryenabled(true)),
- new LoadTypeConfig(new LoadTypeConfig.Builder())),
- factory, docMan, clusterList, new NullFeedMetric(true)));
- }
-
- @Test
- public void testQueryParameters() throws Exception {
- VisitSearcher searcher = create();
- VisitorParameters params = searcher.getVisitorParameters(
- newQuery("visit?visit.selection=id.user=1234&visit.cluster=foobar" +
- "&visit.dataHandler=othercluster&visit.fieldSet=[header]&visit.fromTimestamp=112&visit.toTimestamp=224" +
- "&visit.maxBucketsPerVisitor=2&visit.maxPendingMessagesPerVisitor=7&visit.maxPendingVisitors=14" +
- "&visit.ordering=ASCENDING&priority=NORMAL_1&tracelevel=7&visit.visitInconsistentBuckets&visit.visitRemoves"), null);
-
- assertEquals("id.user=1234", params.getDocumentSelection());
- assertEquals(7, params.getMaxPending());
- assertEquals(2, params.getMaxBucketsPerVisitor());
- assertEquals(14, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount());
- assertEquals("[Storage:cluster=foobar;clusterconfigid=storage/cluster.foobar]", params.getRoute().toString());
- assertEquals("othercluster", params.getRemoteDataHandler());
- assertEquals("[header]", params.fieldSet());
- assertEquals(112, params.getFromTimestamp());
- assertEquals(224, params.getToTimestamp());
- assertEquals(VisitorOrdering.ASCENDING, params.getVisitorOrdering());
- assertEquals(DocumentProtocol.Priority.NORMAL_1, params.getPriority());
- assertEquals(7, params.getTraceLevel());
- assertEquals(true, params.visitInconsistentBuckets());
- assertEquals(true, params.visitRemoves());
- }
-
- @Test
- public void timestampQueryParametersAreParsedAsLongs() throws Exception {
- VisitorParameters params = create().getVisitorParameters(
- newQuery("visit?visit.selection=id.user=1234&" +
- "visit.fromTimestamp=1419021596000000&" +
- "visit.toTimestamp=1419021597000000"), null);
- assertEquals(1419021596000000L, params.getFromTimestamp());
- assertEquals(1419021597000000L, params.getToTimestamp());
- }
-
- @Test
- public void testQueryParametersDefaults() throws Exception {
- VisitSearcher searcher = create();
- VisitorParameters params = searcher.getVisitorParameters(
- newQuery("visit?visit.selection=id.user=1234&hits=100"), null);
-
- assertEquals("id.user=1234", params.getDocumentSelection());
- assertEquals(1, params.getMaxBucketsPerVisitor());
- assertEquals(1, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount());
- assertEquals(1, params.getMaxFirstPassHits());
- assertEquals(1, params.getMaxTotalHits());
- assertEquals(32, params.getMaxPending());
- assertEquals(false, params.visitInconsistentBuckets());
- }
-
- @Test
- public void testWrongCluster() throws Exception {
- VisitSearcher searcher = create();
-
- try {
- searcher.getVisitorParameters(
- newQuery("visit?visit.selection=id.user=1234&visit.cluster=unknown"), null);
-
- assertTrue(false);
- } catch (Exception e) {
- // e.printStackTrace();
- }
- }
-
-
- @Test(expected = IllegalArgumentException.class)
- public void testNoClusterParamWhenSeveralClusters() throws Exception {
- DocumentSessionFactory factory = new DocumentSessionFactory(docType);
- ClusterListConfig.Storage.Builder storageCluster1 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foo").name("foo");
- ClusterListConfig.Storage.Builder storageCluster2 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.bar").name("bar");
- ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(Arrays.asList(storageCluster1, storageCluster2)));
- ClusterList clusterList = new ClusterList(clusterListCfg);
- VisitSearcher searcher = new VisitSearcher(new FeedContext(
- new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(100).route("whatever").retryenabled(true)),
- new LoadTypeConfig(new LoadTypeConfig.Builder())),
- factory, docMan, clusterList, new NullFeedMetric(true)));
-
- searcher.getVisitorParameters(newQuery("visit?visit.selection=id.user=1234"), null);
- }
-
- @Test
- public void testSimple() throws Exception {
- Chain<Searcher> searchChain = new Chain<>(create());
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("visit?visit.selection=id.user=1234&hits=100"));
- assertEquals(1, result.hits().size());
- assertRendered(
- "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<document documenttype=\"kittens\" documentid=\"userdoc:foo:1234:bar\"/>\n" +
- "</result>\n", result);
- }
-
- private Result invokeVisitRemovesSearchChain() throws Exception {
- Chain<Searcher> searchChain = new Chain<>(create());
- return new Execution(searchChain, Execution.Context.createContextStub()).search(
- newQuery("visit?visit.selection=id.user=1234&hits=100&visit.visitRemoves=true"));
- }
-
- @Test
- public void visitRemovesIncludesRemoveEntriesInResultXml() throws Exception {
- Result result = invokeVisitRemovesSearchChain();
- assertEquals(2, result.hits().size());
- assertRendered(
- "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<document documenttype=\"kittens\" documentid=\"userdoc:foo:1234:bar\"/>\n" +
- "<remove documentid=\"userdoc:foo:1234:removed\"/>\n" +
- "</result>\n", result);
- }
-
- @Test
- public void removedDocumentIdsAreXmlEscaped() throws Exception {
- factory = mock(DocumentSessionFactory.class);
- when(factory.createVisitorSession(any(VisitorParameters.class))).thenAnswer((p) -> {
- VisitorParameters params = (VisitorParameters)p.getArguments()[0];
- DummyVisitorSession session = new DummyVisitorSession(params, docType);
- session.clearAutoReplyMessages();
- session.addRemoveReply("userdoc:foo:1234:<rem\"o\"ved&stuff>");
- return session;
- });
- Result result = invokeVisitRemovesSearchChain();
- assertEquals(1, result.hits().size());
- assertRendered(
- "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "<result>\n" +
- "<remove documentid=\"userdoc:foo:1234:&lt;rem&quot;o&quot;ved&amp;stuff&gt;\"/>\n" +
- "</result>\n", result);
- }
-
- private Result invokeSearcherWithUserQuery() throws Exception {
- Chain<Searcher> searchChain = new Chain<>(create());
- return new Execution(searchChain, Execution.Context.createContextStub())
- .search(new Query("visit?visit.selection=id.user=1234&hits=100"));
- }
-
- @Test
- public void waitUntilDoneFailureReturnsTimeoutErrorHit() throws Exception {
- VisitorSession session = mock(VisitorSession.class);
- when(session.waitUntilDone(anyLong())).thenReturn(false);
- factory = mock(DocumentSessionFactory.class);
- when(factory.createVisitorSession(any(VisitorParameters.class))).thenReturn(session);
-
- Result result = invokeSearcherWithUserQuery();
- assertNotNull(result.hits().getErrorHit());
- assertEquals(Error.TIMEOUT.code, result.hits().getErrorHit().errors().iterator().next().getCode());
- }
-
- @Test
- @SuppressWarnings("deprecation")
- public void testRendererWiring() throws Exception {
- Chain<Searcher> searchChain = new Chain<>(create());
- {
- Query query = newQuery("visit?visit.selection=id.user=1234&hits=100&format=json");
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query);
- assertEquals(com.yahoo.prelude.templates.DefaultTemplateSet.class, result.getTemplating().getTemplates().getClass());
- }
- {
- Query query = newQuery("visit?visit.selection=id.user=1234&hits=100&format=JsonRenderer");
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query);
- assertEquals(com.yahoo.prelude.templates.DefaultTemplateSet.class, result.getTemplating().getTemplates().getClass());
- }
- {
- Query query = newQuery("visit?visit.selection=id.user=1234&hits=100");
- Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query);
- assertEquals(DocumentXMLTemplate.class, result.getTemplating().getTemplates().getClass());
- }
- }
-
- public static void assertRendered(String expected, Result result) throws Exception {
- assertEquals(expected, ResultRenderingUtil.getRendered(result));
- }
-
- private Query newQuery(String queryString) {
- return new Query(HttpRequest.createTestRequest(queryString, com.yahoo.jdisc.http.HttpRequest.Method.GET));
- }
-
-}