diff options
Diffstat (limited to 'vespaclient-container-plugin/src/test/java')
9 files changed, 1 insertions, 2754 deletions
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java index b4067c3fe78..6bca3637297 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java @@ -32,4 +32,5 @@ public class DocumentApiApplicationTest { socket.close(); return port; } + } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore b/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore +++ /dev/null diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java deleted file mode 100755 index 2c99ec194c3..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java +++ /dev/null @@ -1,1064 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import com.yahoo.component.ComponentId; -import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.docproc.CallStack; -import com.yahoo.jdisc.HeaderFields; -import com.yahoo.messagebus.*; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.container.Container; -import com.yahoo.docproc.*; -import com.yahoo.docproc.jdisc.DocumentProcessingHandler; -import com.yahoo.docproc.jdisc.DocumentProcessingHandlerParameters; -import com.yahoo.document.*; -import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; -import com.yahoo.documentapi.messagebus.protocol.*; -import com.yahoo.feedapi.DummySessionFactory; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.jdisc.handler.RequestHandler; -import com.yahoo.messagebus.routing.Route; -import com.yahoo.vespaclient.ClusterDef; -import com.yahoo.vespaclient.ClusterList; -import com.yahoo.vespaclient.config.FeederConfig; -import org.junit.After; -import org.junit.Test; - -import java.io.*; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; -import java.util.logging.Logger; -import java.util.zip.GZIPOutputStream; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@SuppressWarnings("deprecation") // VespaFeedHandler classes are going away on Vespa 7 -public class VespaFeedHandlerTestCase { - - private VespaFeedHandler feedHandler; - private VespaFeedHandlerRemove removeHandler; - private VespaFeedHandlerStatus statusHandler; - private VespaFeedHandlerRemoveLocation removeLocationHandler; - private FeedContext context; - - private DummySessionFactory factory; - private final String xmlFilesPath = "src/test/files/feedhandler/"; - - public void setup(com.yahoo.messagebus.Error e, LoadTypeConfig loadTypeCfg, - boolean autoReply, - DummySessionFactory.ReplyFactory autoReplyFactory) throws Exception { - DocumentTypeManager docMan = new DocumentTypeManager(); - DocumentTypeManagerConfigurer.configure(docMan, "file:" + xmlFilesPath + "documentmanager.cfg"); - - if (autoReply) { - if (autoReplyFactory != null) { - factory = DummySessionFactory.createWithAutoReplyFactory(autoReplyFactory); - } else { - factory = DummySessionFactory.createWithErrorAutoReply(e); - } - } else { - factory = DummySessionFactory.createDefault(); - } - - context = new FeedContext(new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder()), loadTypeCfg), factory, docMan, new ClusterList(), new NullFeedMetric(true)); - - Executor threadPool = Executors.newCachedThreadPool(); - feedHandler = new VespaFeedHandler(context, threadPool); - removeHandler = new VespaFeedHandlerRemove(context, threadPool); - statusHandler = new VespaFeedHandlerStatus(context, false, false, threadPool); - removeLocationHandler = new VespaFeedHandlerRemoveLocation(context, threadPool); - - CallStack dpCallstack = new CallStack("bar"); - dpCallstack.addLast(new TestDocProc()); - dpCallstack.addLast(new TestLaterDocProc()); - - DocprocService myservice = new DocprocService("bar"); - myservice.setCallStack(dpCallstack); - myservice.setInService(true); - - ComponentRegistry<DocprocService> registry = new ComponentRegistry<DocprocService>(); - registry.register(new ComponentId(myservice.getName()), myservice); - - DocumentProcessingHandler handler = new DocumentProcessingHandler(registry, - new ComponentRegistry<>(), new ComponentRegistry<>(), - new DocumentProcessingHandlerParameters()); - - Container container = Container.get(); - ComponentRegistry<RequestHandler> requestHandlerComponentRegistry = new ComponentRegistry<>(); - requestHandlerComponentRegistry.register(new ComponentId(DocumentProcessingHandler.class.getName()), handler); - container.setRequestHandlerRegistry(requestHandlerComponentRegistry); - } - - public void setup(com.yahoo.messagebus.Error e) throws Exception { - setup(e, new LoadTypeConfig(new LoadTypeConfig.Builder()), true, null); - } - - public void setupWithReplyFactory(DummySessionFactory.ReplyFactory autoReplyFactory) throws Exception { - setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder()), true, autoReplyFactory); - } - - public void setup() throws Exception { - setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder()), false, null); - } - - @After - public void resetContainer() { - Container.resetInstance(); - } - - - @Test - public void testLoadTypes() throws Exception { - List<LoadTypeConfig.Type.Builder> typeBuilder = new ArrayList<>(); - typeBuilder.add(new LoadTypeConfig.Type.Builder().id(1234).name("foo").priority("VERY_LOW")); - typeBuilder.add(new LoadTypeConfig.Type.Builder().id(4567).name("bar").priority("NORMAL_3")); - - setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder().type(typeBuilder)), true, null); - - { - Result res = testRequest(HttpRequest.createTestRequest("remove?id=doc:test:removeme&loadtype=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:removeme", d.toString()); - assertEquals(new LoadType(1234, "foo", DocumentProtocol.Priority.VERY_LOW), ((DocumentMessage)m).getLoadType()); - assertEquals(DocumentProtocol.Priority.VERY_LOW, ((DocumentMessage)m).getPriority()); - } - } - - @Test - public void testPostXML() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - assertTrue(res.output.contains("count=\"2\"")); - assertTrue(res.error == null); - } - - @Test - public void testPostXMLAsync() throws Exception { - setup(); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?asynchronous=true"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - // Should not have metrics at this point. - assertTrue(!res.output.contains("count=\"2\"")); - assertTrue(res.error == null); - } - - - @Test - public void testPostGZIPedXML() throws Exception { - setup(null); - Result res = testFeedGZIP(xmlFilesPath + "test10b.xml", "feed?"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - assertTrue(res.error == null); - } - - @Test - public void testDocProc() throws Exception { - setup(null); - - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - Document d = ((PutDocumentMessage)m).getDocumentPut().getDocument(); - - assertEquals("doc:news:http://news10a", d.getId().toString()); - assertEquals(new IntegerFieldValue(1234), d.getFieldValue("last_downloaded")); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - Document d = ((PutDocumentMessage)m).getDocumentPut().getDocument(); - - assertEquals("doc:news:http://news10b", d.getId().toString()); - assertEquals(new IntegerFieldValue(1234), d.getFieldValue("last_downloaded")); - } - } - - @Test - public void testPostXMLVariousTypes() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10.xml", "feed?"); - - assertEquals(5, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - { - Message m = res.messages.get(2); - assertEquals(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, m.getType()); - DocumentId d = ((UpdateDocumentMessage)m).getDocumentUpdate().getId(); - assertEquals("doc:news:http://news10c", d.toString()); - } - { - Message m = res.messages.get(3); - assertEquals(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, m.getType()); - DocumentId d = ((UpdateDocumentMessage)m).getDocumentUpdate().getId(); - assertEquals("doc:news:http://news10d", d.toString()); - } - { - Message m = res.messages.get(4); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:news:http://news10e", d.toString()); - } - - String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); - - assertEquals("<result>\n" + - "\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <latency count=\"5\"/>\n" + - " <count count=\"5\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"1\"/>\n" + - " <count count=\"1\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - "\n" + - "</result>\n", val); - } - - @Test - public void testStatusPage() throws Exception { - setup(null); - - testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); - testFeed(xmlFilesPath + "test10.xml", "feed?"); - testFeed(xmlFilesPath + "test10.xml", "feed?route=storage"); - testFeed(xmlFilesPath + "test_removes", "remove?"); - - assertEquals(2, factory.sessionsCreated()); - Result res = testRequest(HttpRequest.createTestRequest("feedstatus?", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - - String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); - val = val.replaceAll("to=\"[0-9]*\"", "to=\"0\""); - - assertEquals("<status>\n" + - "\n" + - " <snapshot name=\"Total metrics from start until current time\" from=\"0\" to=\"0\" period=\"0\">\n" + - " <routes>\n" + - " <route name=\"total\">\n" + - " <total>\n" + - " <latency count=\"14\"/>\n" + - " <count count=\"14\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"6\"/>\n" + - " <count count=\"6\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <latency count=\"9\"/>\n" + - " <count count=\"9\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"3\"/>\n" + - " <count count=\"3\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " <route name=\"storage\">\n" + - " <total>\n" + - " <latency count=\"5\"/>\n" + - " <count count=\"5\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"1\"/>\n" + - " <count count=\"1\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " </routes>\n" + - " </snapshot>\n" + - "\n" + - "</status>\n", val); - } - - @Test - public void testStatusPage2() throws Exception { - setup(null); - - testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); - testFeed(xmlFilesPath + "test10.xml", "feed?"); - testFeed(xmlFilesPath + "test10.xml", "feed?route=storage"); - testFeed(xmlFilesPath + "test_removes", "remove?"); - - assertEquals(2, factory.sessionsCreated()); - Result res = testRequest(HttpRequest.createTestRequest("feed?status", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - - String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); - val = val.replaceAll("to=\"[0-9]*\"", "to=\"0\""); - - assertEquals("<status>\n" + - "\n" + - " <routes>\n" + - " <route name=\"total\" description=\"Messages sent to all routes\">\n" + - " <total description=\"All kinds of messages sent to the given route\">\n" + - " <latency count=\"14\"/>\n" + - " <count count=\"14\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"6\"/>\n" + - " <count count=\"6\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " <route name=\"default\" description=\"Messages sent to the named route\">\n" + - " <total description=\"All kinds of messages sent to the given route\">\n" + - " <latency count=\"9\"/>\n" + - " <count count=\"9\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"3\"/>\n" + - " <count count=\"3\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " <route name=\"storage\" description=\"Messages sent to the named route\">\n" + - " <total description=\"All kinds of messages sent to the given route\">\n" + - " <latency count=\"5\"/>\n" + - " <count count=\"5\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"1\"/>\n" + - " <count count=\"1\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " </routes>\n" + - "\n" + - "</status>\n", val); - } - - @Test - public void testMetricForIgnoredDocumentsIsIncreased() throws Exception { - DummySessionFactory.ReplyFactory replyFactory = new DummySessionFactory.ReplyFactory() { - @Override - public Reply createReply(Message m) { - return new DocumentIgnoredReply(); - } - }; - setupWithReplyFactory(replyFactory); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); - assertEquals(2, res.messages.size()); - - String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); - - assertEquals("<result>\n" + - "\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <ignored count=\"2\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <ignored count=\"2\"/>\n" + - " </putdocument>\n" + - " </route>\n" + - "\n" + - "</result>\n", val); - } - - @Test - public void testPostXMLWithMBusFailureAllowed() throws Exception { - setup(new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_BUCKET_DELETED, "Hello world in <document>")); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?abortonfeederror=false"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - String val = res.output.replaceAll("average=\"[0-9]*\" last=\"[0-9]*\" min=\"[0-9]*\" max=\"[0-9]*\" ", ""); - System.out.println(val); - - assertEquals("<result>\n" + - "\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <errors>\n" + - " <error name=\"total\" count=\"2\"/>\n" + - " <error name=\"BUCKET_DELETED\" count=\"2\"/>\n" + - " </errors>\n" + - " </total>\n" + - " <putdocument>\n" + - " <errors>\n" + - " <error name=\"total\" count=\"2\"/>\n" + - " <error name=\"BUCKET_DELETED\" count=\"2\"/>\n" + - " </errors>\n" + - " </putdocument>\n" + - " </route>\n\n" + - " <errors count=\"2\">\n" + - " <error message=\"PUT[doc:news:http://news10a] [BUCKET_DELETED] Hello world in <document>\"/>\n" + - " <error message=\"PUT[doc:news:http://news10b] [BUCKET_DELETED] Hello world in <document>\"/>\n" + - " </errors>\n" + - "\n" + - "</result>\n", val); - - assertTrue(res.error != null); - assertTrue(res.errorCount > 0); - } - - @Test - public void testPostXMLWithMBusFailure() throws Exception { - setup(new com.yahoo.messagebus.Error(32, "Hello world")); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); - - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - - String val = res.output.replaceAll("average=\"[0-9]*\" last=\"[0-9]*\" min=\"[0-9]*\" max=\"[0-9]*\" ", ""); - assertEquals("<result>\n" + - "\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <errors>\n" + - " <error name=\"total\" count=\"1\"/>\n" + - " <error name=\"UNKNOWN(32)\" count=\"1\"/>\n" + - " </errors>\n" + - " </total>\n" + - " <putdocument>\n" + - " <errors>\n" + - " <error name=\"total\" count=\"1\"/>\n" + - " <error name=\"UNKNOWN(32)\" count=\"1\"/>\n" + - " </errors>\n" + - " </putdocument>\n" + - " </route>\n\n" + - " <errors count=\"1\">\n" + - " <error message=\"PUT[doc:news:http://news10a] [UNKNOWN(32)] Hello world\"/>\n" + - " </errors>\n" + - "\n" + - "</result>\n", val); - - assertTrue(res.error != null); - assertTrue(res.errorCount > 0); - } - - @Test - public void testPostXMLWithIllegalDocId() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_docid.xml", "feed?"); - - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - } - - @Test - public void testPostXMLWithIllegalDocIdAllowFailure() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_docid.xml", "feed?abortondocumenterror=false"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:ok", d.toString()); - } - } - - @Test - public void testPostUnparseableXML() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_xml.xml", "feed?"); - - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - } - - @Test - public void testOverrides() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222&route=storage&priority=HIGH_2&totaltimeout=-1"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertEquals(2222, m.getTimeRemaining()); - assertEquals(Route.parse("storage"), m.getRoute()); - assertEquals(DocumentProtocol.Priority.HIGH_2, ((DocumentMessage)m).getPriority()); - } - } - - @Test - public void testTimeoutWithNoUpperBound() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222&totaltimeout=-1"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertEquals(2222, m.getTimeRemaining()); - } - } - - @Test - public void testTimeout() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertTrue(2222 >= m.getTimeRemaining()); - } - } - - @Test - public void testTotalTimeout() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?totaltimeout=2.222"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertTrue(2222 >= m.getTimeRemaining()); - } - } - - @Test - public void testTotalTimeoutAndNormalTimeout() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?totaltimeout=1000&timeout=2.222"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertEquals(2222, m.getTimeRemaining()); - } - } - - @Test - public void testBogusPriority() throws Exception { - try { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2222&route=storage&priority=HIPSTER_DOOFUS"); - assertTrue(false); - } catch (IllegalArgumentException e) { - } - } - - @Test - public void testPostXMLWithIllegalDocIdFirst() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_docid_first.xml", "feed?"); - - assertEquals(0, res.messages.size()); - } - - @Test - public void testPostXMLWithIllegalDocIdFirstNoAbort() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_docid_first.xml", "feed?abortondocumenterror=false"); - - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - } - - @Test - public void testSimpleRemove() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id=doc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:removeme", d.toString()); - } - } - - @Test - public void testRemoveUser() throws Exception { - setup(null); - - context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?user=1234", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); - String selection = ((RemoveLocationMessage)m).getDocumentSelection(); - assertEquals("storage", m.getRoute().toString()); - assertEquals("id.user=1234", selection); - } - } - - @Test - public void testRemoveGroup() throws Exception { - setup(null); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); - String selection = ((RemoveLocationMessage)m).getDocumentSelection(); - assertEquals("storage", m.getRoute().toString()); - assertEquals("id.group=\"foo\"", selection); - } - } - - @Test - public void testRemoveBadSyntax() throws Exception { - setup(null); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo&user=12345", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(0, res.messages.size()); - assertTrue(res.error.toString().contains("Exactly one of")); - } - - @Test - public void testRemoveGroupMultipleClusters() throws Exception { - setup(null); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage1", "storage/cluster.storage1")); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage2", "storage/cluster.storage2")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(0, res.messages.size()); - assertTrue(res.error.toString().contains("More than one")); - } - - @Test - public void testRemoveGroupNoClusters() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(0, res.messages.size()); - assertTrue(res.error.toString().contains("No storage clusters")); - } - - @Test - public void testRemoveSelection() throws Exception { - setup(null); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?selection=id.user=1234", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); - String selection = ((RemoveLocationMessage)m).getDocumentSelection(); - assertEquals("id.user=1234", selection); - } - } - - @Test - public void testSimpleRemoveIndex() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=doc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:removeme", d.toString()); - } - } - - @Test - public void testPostRemove() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_removes", "remove?"); - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:remove1", d.toString()); - } - - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:remove2", d.toString()); - } - } - - @Test - public void testRemoveBogusId() throws Exception { - try { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id=unknowndoc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertTrue(false); - } catch (Exception e) { - } - } - - @Test - public void testMultiRemove() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=doc:test:removeme&id[1]=doc:test:remove2&id[2]=doc:test:remove3", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(3, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:removeme", d.toString()); - } - - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:remove2", d.toString()); - } - - { - Message m = res.messages.get(2); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:remove3", d.toString()); - } - } - - @Test - public void testMultiRemoveSameDoc() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=userdoc:footype:1234:foo&id[1]=userdoc:footype:1234:foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - } - - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - } - } - - @Test - public void testFeedHandlerStatusCreation() throws Exception { - VespaFeedHandlerStatus status = new VespaFeedHandlerStatus( - new FeedContext(new MessagePropertyProcessor( - new FeederConfig(new FeederConfig.Builder()), - new LoadTypeConfig(new LoadTypeConfig.Builder())), - factory, null, new ClusterList(), new NullFeedMetric(true)), - true, true, - Executors.newCachedThreadPool()); - } - - private class TestDocProc extends DocumentProcessor { - @Override - public Progress process(Processing processing) { - for (DocumentOperation op : processing.getDocumentOperations()) { - if (op instanceof DocumentPut) { - Document document = ((DocumentPut)op).getDocument(); - document.setFieldValue("last_downloaded", new IntegerFieldValue(1234)); - } - } - return Progress.DONE; - } - } - - private class TestLaterDocProc extends DocumentProcessor { - private final Logger log = Logger.getLogger(TestLaterDocProc.class.getName()); - - private int counter = 0; - @Override - public Progress process(Processing processing) { - synchronized (this) { - counter++; - if (counter % 2 == 1) { - log.info("Returning LATER."); - return Progress.LATER; - } - log.info("Returning DONE."); - return Progress.DONE; - } - } - } - - private Result testRequest(HttpRequest req) throws Exception { - HttpResponse response = null; - String feedPrefix = "feed"; - String removePrefix = "remove"; - String feedStatusPrefix = "feedstatus"; - String removeLocationPrefix = "removelocation"; - - if (req.getUri().getPath().startsWith(feedPrefix)) { - response = feedHandler.handle(req); - } - if (req.getUri().getPath().startsWith(removePrefix)) { - response = removeHandler.handle(req); - } - if (req.getUri().getPath().startsWith(feedStatusPrefix)) { - response = statusHandler.handle(req); - } - if (req.getUri().getPath().startsWith(removeLocationPrefix)) { - response = removeLocationHandler.handle(req); - } - - ByteArrayOutputStream output = new ByteArrayOutputStream(); - response.render(output); - - Result res = new Result(); - res.messages = factory.messages; - res.output = new String(output.toByteArray()); - - if (response instanceof FeedResponse) { - FeedResponse feedResponse = (FeedResponse)response; - res.error = feedResponse.getErrorMessageList().isEmpty() ? null : feedResponse.getErrorMessageList().get(0); - res.errorCount = feedResponse.getErrorMessageList().size(); - assertTrue(feedResponse.isSuccess() == (res.errorCount == 0)); - } - return res; - } - - private Result testFeed(String xmlFile, String request) throws Exception { - return testRequest(new FileRequest(new File(xmlFile), request).toRequest()); - } - - private Result testFeedGZIP(String xmlFile, String request) throws Exception { - return testRequest(new FileRequest(new File(xmlFile), request, true).toRequest()); - } - - private class FileRequest { - - private final String req; - private final File f; - private boolean gzip = false; - - FileRequest(File f, String req) { - this.req = req; - this.f = f; - } - - FileRequest(File f, String req, boolean gzip) { - this.f = f; - this.req = req; - this.gzip = gzip; - } - - public InputStream getData() { - try { - InputStream fileStream = new FileInputStream(f); - if (gzip) { - // Not exactly pretty, but in lack of an elegant way of transcoding - ByteArrayOutputStream rawOut = new ByteArrayOutputStream(); - GZIPOutputStream compressed = new GZIPOutputStream(rawOut); - byte[] buffer = new byte[1024]; - int read = -1; - while (true) { - read = fileStream.read(buffer); - if (read == -1) break; - compressed.write(buffer, 0, read); - } - compressed.finish(); - compressed.flush(); - rawOut.flush(); - return new ByteArrayInputStream(rawOut.toByteArray()); - } - return fileStream; - } catch (Exception e) { - return null; - } - } - - public void addHeaders(HeaderFields headers) { - headers.add("Content-Type", "image/jpeg"); - if (gzip) - headers.add("Content-Encoding", "gzip"); - } - - public HttpRequest toRequest() { - HttpRequest request = HttpRequest.createTestRequest(req, com.yahoo.jdisc.http.HttpRequest.Method.GET, getData()); - addHeaders(request.getJDiscRequest().headers()); - return request; - } - - } - - private class Result { - private List<Message> messages; - private String output; - private com.yahoo.processing.request.ErrorMessage error; - private int errorCount; - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java deleted file mode 100644 index b7a911109cc..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.BucketId; -import com.yahoo.documentapi.ProgressToken; -import com.yahoo.documentapi.VisitorIterator; -import org.junit.Test; - -import java.util.Set; -import java.util.TreeSet; - -import static org.junit.Assert.*; - -@SuppressWarnings("deprecation") -public class ContinuationHitTest { - - private static final String SINGLE_BUCKET_URL_SAFE_BASE64 - = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAHqNFZ4mrz-_wAAAAAAAAAA"; - private static final String MULTI_BUCKET_URL_SAFE_BASE64 - = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz--gAAAAAAAAAA6" + - "jRWeJq8_vsAAAAAAAAAAOo0VniavP7_AAAAAAAAAAA="; - - @Test - public void continuationTokensAreUrlSafeBase64Encoded() throws Exception { - ContinuationHit hit = new ContinuationHit(createSingleBucketProgress()); - // We want -_ instead of +/ - assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, hit.getValue()); - } - - @Test - public void continuationTokensAreNotBrokenIntoMultipleLines() throws Exception { - ContinuationHit hit = new ContinuationHit(createMultiBucketProgress()); - assertTrue(hit.getValue().length() > 76); // Ensure we exceed MIME line length limits. - assertFalse(hit.getValue().contains("\n")); - } - - @Test - public void decodingAcceptsUrlSafeTokens() throws Exception { - final ProgressToken token = ContinuationHit.getToken(SINGLE_BUCKET_URL_SAFE_BASE64); - // Roundtrip should yield identical results. - assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, - new ContinuationHit(token).getValue()); - } - - /** - * Legacy Base64 encoder emitted MIME Base64. Ensure we handle tokens from that era. - */ - @Test - public void decodingAcceptsLegacyNonUrlSafeTokens() throws Exception { - final String legacyBase64 = convertedToMimeBase64Chars(SINGLE_BUCKET_URL_SAFE_BASE64); - final ProgressToken legacyToken = ContinuationHit.getToken(legacyBase64); - - assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, - new ContinuationHit(legacyToken).getValue()); - } - - /** - * Legacy Base64 encoder would happily output line breaks after each MIME line - * boundary. Ensure we handle these gracefully. - */ - @Test - public void decodingAcceptsLegacyMimeLineBrokenTokens() throws Exception { - final String multiBucketLegacyToken = - "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz++gAAAAAAAAAA6jRWeJq8/vsA\r\n" + - "AAAAAAAAAOo0VniavP7/AAAAAAAAAAA="; - final ProgressToken legacyToken = ContinuationHit.getToken(multiBucketLegacyToken); - - assertEquals(MULTI_BUCKET_URL_SAFE_BASE64, - new ContinuationHit(legacyToken).getValue()); - } - - /** - * Returns a ProgressToken whose base 64 representation will be _less_ than 76 bytes (MIME line limit) - */ - private ProgressToken createSingleBucketProgress() { - ProgressToken token = new ProgressToken(16); - // Use explicit bucket set so we can better control the binary representation - // of the buckets, and thus the values written as base 64. - Set<BucketId> buckets = new TreeSet<>(); - // This particular bucket ID will contain +/ chars when output as non-URL safe base 64. - buckets.add(new BucketId(58, 0x123456789abcfeffL)); - VisitorIterator.createFromExplicitBucketSet(buckets, 16, token); // "Prime" the token. - return token; - } - - /** - * Returns a ProgressToken whose base 64 representation will be _more_ than 76 bytes (MIME line limit) - */ - private ProgressToken createMultiBucketProgress() { - ProgressToken token = new ProgressToken(16); - Set<BucketId> buckets = new TreeSet<>(); - buckets.add(new BucketId(58, 0x123456789abcfeffL)); - buckets.add(new BucketId(58, 0x123456789abcfefaL)); - buckets.add(new BucketId(58, 0x123456789abcfefbL)); - VisitorIterator.createFromExplicitBucketSet(buckets, 16, token); // "Prime" the token. - return token; - } - - private String convertedToMimeBase64Chars(String token) { - // Doesn't split on MIME line boundaries, so not fully MIME compliant. - return token.replace('-', '+').replace('_', '/'); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java deleted file mode 100755 index 2015efa4d5d..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentType; -import com.yahoo.documentapi.VisitorParameters; -import com.yahoo.documentapi.VisitorSession; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; -import com.yahoo.feedapi.DummySessionFactory; -import com.yahoo.feedapi.SendSession; -import com.yahoo.jdisc.Metric; -import com.yahoo.messagebus.*; -import com.yahoo.messagebus.Error; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.HashMap; - -/** - * Used to automatically reply with a GetDocumentReply for every received GetDocumentMessage - */ -public class DocumentSessionFactory extends DummySessionFactory { - - private DocumentType docType; - private Error error; - // Reply instances are shared between the two collections. - List<GetDocumentReply> autoReplies; - Map<DocumentId, GetDocumentReply> autoReplyLookup = new HashMap<>(); - boolean autoReply = true; - boolean nullReply = false; - private int sessionsCreated = 0; - - private class DocumentReplySession extends SendSession { - - ReplyHandler handler; - Error e; - DummySessionFactory owner; - - public DocumentReplySession(ReplyHandler handler, Error e, DummySessionFactory owner) { - this.handler = handler; - this.e = e; - this.owner = owner; - } - - protected Result onSend(Message m, boolean blockIfQueueFull) throws InterruptedException { - if (!(m instanceof GetDocumentMessage)) { - throw new IllegalArgumentException("Expected GetDocumentMessage"); - } - GetDocumentMessage gm = (GetDocumentMessage)m; - owner.messages.add(m); - if (autoReply) { - Document replyDoc; - if (!nullReply) { - replyDoc = new Document(docType, gm.getDocumentId()); - } else { - replyDoc = null; - } - Reply r = new GetDocumentReply(replyDoc); - r.setMessage(m); - r.setContext(m.getContext()); - if (e != null) { - r.addError(e); - } - handler.handleReply(r); - } else if (owner.messages.size() == autoReplies.size()) { - // Pair up all replies with their messages - for (Message msg : owner.messages) { - GetDocumentReply reply = autoReplyLookup.get(((GetDocumentMessage)msg).getDocumentId()); - reply.setMessage(msg); - reply.setContext(msg.getContext()); - if (e != null) { - reply.addError(e); - } - } - // Now send them in the correct order. Instances are shared, so source - // messages and contexts are properly set - for (Reply reply : autoReplies) { - handler.handleReply(reply); - } - } - - return Result.ACCEPTED; - } - - public void close() { - } - } - - public DocumentSessionFactory(DocumentType docType) { - this.docType = docType; - this.error = null; - } - - public DocumentSessionFactory(DocumentType docType, Error error, boolean autoReply, GetDocumentReply... autoReplies) { - this.docType = docType; - this.error = error; - this.autoReplies = Arrays.asList(autoReplies); - for (GetDocumentReply reply : autoReplies) { - this.autoReplyLookup.put(reply.getDocument().getId(), reply); - } - this.autoReply = autoReply; - } - - public boolean isNullReply() { - return nullReply; - } - - public void setNullReply(boolean nullReply) { - this.nullReply = nullReply; - } - - public int getSessionsCreated() { - return sessionsCreated; - } - - public SendSession createSendSession(ReplyHandler r, Metric metric) { - ++sessionsCreated; - return new DocumentReplySession(r, error, this); - } - - @Override - public VisitorSession createVisitorSession(VisitorParameters p) { - return new DummyVisitorSession(p, docType); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java deleted file mode 100644 index 02d26f8bd93..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.DocumentType; -import com.yahoo.documentapi.*; -import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.Trace; - -import java.util.ArrayList; -import java.util.List; - -/** - * Stub to test visitors. - */ -public class DummyVisitorSession implements VisitorSession { - - final VisitorParameters parameters; - final DocumentType documentType; - final List<Message> autoReplyMessages = new ArrayList<>(); - - DummyVisitorSession(VisitorParameters p, DocumentType documentType) { - parameters = p; - this.documentType = documentType; - p.getLocalDataHandler().setSession(this); - addDefaultReplyMessages(); - } - - @Override - public boolean isDone() { - return true; - } - - @Override - public ProgressToken getProgress() { - return new ProgressToken(12); - } - - @Override - public Trace getTrace() { - return null; - } - - public void addDocumentReply(String docId) { - Document replyDoc = new Document(documentType, docId); - autoReplyMessages.add(new PutDocumentMessage(new DocumentPut(replyDoc))); - } - - public void addRemoveReply(String docId) { - autoReplyMessages.add(new RemoveDocumentMessage(new DocumentId(docId))); - } - - public void addDefaultReplyMessages() { - addDocumentReply("userdoc:foo:1234:bar"); - if (parameters.visitRemoves()) { - addRemoveReply("userdoc:foo:1234:removed"); - } - } - - public void clearAutoReplyMessages() { - autoReplyMessages.clear(); - } - - @Override - public boolean waitUntilDone(long l) throws InterruptedException { - for (Message msg : autoReplyMessages) { - parameters.getLocalDataHandler().onMessage(msg, new AckToken(this)); - } - return true; - } - - @Override - public void ack(AckToken ackToken) { - } - - @Override - public void abort() { - } - - @Override - public VisitorResponse getNext() { - return null; - } - - @Override - public VisitorResponse getNext(int i) throws InterruptedException { - return null; - } - - @Override - public void destroy() { - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java deleted file mode 100755 index 14fe0dc857d..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java +++ /dev/null @@ -1,1093 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.component.chain.Chain; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.feedhandler.NullFeedMetric; -import com.yahoo.jdisc.HeaderFields; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.document.*; -import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.document.datatypes.Raw; -import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.routing.Route; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.vespaclient.ClusterList; -import com.yahoo.vespaclient.config.FeederConfig; - -import org.junit.Test; - -import java.io.*; -import java.nio.ByteBuffer; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.zip.GZIPOutputStream; - -import static org.junit.Assert.*; - -@SuppressWarnings("deprecation") -public class GetSearcherTestCase { - - private DocumentTypeManager docMan = null; - private DocumentType docType; - private FeederConfig defFeedCfg = new FeederConfig(new FeederConfig.Builder()); - private LoadTypeConfig defLoadTypeCfg = new LoadTypeConfig(new LoadTypeConfig.Builder()); - - @org.junit.Before - public void setUp() { - docMan = new DocumentTypeManager(); - docType = new DocumentType("kittens"); - docType.addHeaderField("name", DataType.STRING); - docType.addField("description", DataType.STRING); - docType.addField("image", DataType.STRING); - docType.addField("fluffiness", DataType.INT); - docType.addField("foo", DataType.RAW); - docMan.registerDocumentType(docType); - } - - @org.junit.After - public void tearDown() { - docMan = null; - docType = null; - } - - private void assertHits(HitGroup hits, String... wantedHits) { - assertEquals(wantedHits.length, hits.size()); - for (int i = 0; i < wantedHits.length; ++i) { - assertTrue(hits.get(i) instanceof DocumentHit); - DocumentHit hit = (DocumentHit)hits.get(i); - assertEquals(wantedHits[i], hit.getDocument().getId().toString()); - } - } - - @Test - public void testGetSingleDocumentQuery() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:1:2")); - System.out.println("HTTP request is " + result.getQuery().getHttpRequest()); - - assertEquals(1, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - } - assertEquals(1, result.hits().size()); - assertHits(result.hits(), "userdoc:kittens:1:2"); - // By default, document hit should not have its hit fields set - DocumentHit hit = (DocumentHit)result.hits().get(0); - assertEquals(0, hit.fieldKeys().size()); - } - - @Test - public void testGetMultipleDocumentsQuery() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Query query = newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - - assertEquals(2, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - } - - { - Message m = factory.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:3:4", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - } - assertEquals(2, result.hits().size()); - assertNull(result.hits().getErrorHit()); - assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4"); - assertEquals(2, query.getHits()); - } - - // Test that you can use both query string and POSTed IDs - @Test - public void testGetMultipleDocumentsQueryAndPOST() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - String data = "userdoc:kittens:5:6\nuserdoc:kittens:7:8\nuserdoc:kittens:9:10"; - MockHttpRequest request = new MockHttpRequest(data.getBytes("utf-8"), "/get/?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4"); - Query query = new Query(request.toRequest()); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - - assertEquals(5, factory.messages.size()); - assertEquals(5, result.hits().size()); - assertNull(result.hits().getErrorHit()); - assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4", - "userdoc:kittens:5:6", "userdoc:kittens:7:8", "userdoc:kittens:9:10"); - } - - @Test - public void testGetMultipleDocumentsQueryAndGZippedPOST() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - String data = "userdoc:kittens:5:6\nuserdoc:kittens:7:8\nuserdoc:kittens:9:10"; - - // Create with automatic GZIP encoding - MockHttpRequest request = new MockHttpRequest(data.getBytes("utf-8"), "/get/?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4", true); - Query query = new Query(request.toRequest()); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - - assertEquals(5, factory.messages.size()); - assertEquals(5, result.hits().size()); - assertNull(result.hits().getErrorHit()); - assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4", - "userdoc:kittens:5:6", "userdoc:kittens:7:8", "userdoc:kittens:9:10"); - } - - /* Test that a query without any ids is passed through to the next chain */ - @Test - public void testQueryPassThrough() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - HitGroup hits = new HitGroup("mock"); - hits.add(new Hit("blernsball")); - Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(hits)); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?flarn=blern")); - - assertEquals(0, factory.messages.size()); - assertEquals(1, result.hits().size()); - assertNotNull(result.hits().get("blernsball")); - } - - /* Test that a query will contain both document hits and hits from a searcher - * further down the chain, iff the searcher returns a DocumentHit. - */ - @Test - public void testQueryPassThroughAndGet() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:1234:foo")); - doc1.setFieldValue("name", new StringFieldValue("megacat")); - doc1.setFieldValue("description", new StringFieldValue("supercat")); - doc1.setFieldValue("fluffiness", new IntegerFieldValue(10000)); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1) - }; - - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - DocumentHit backendHit = new DocumentHit(new Document(docType, new DocumentId("userdoc:kittens:5678:bar")), 5); - Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(backendHit)); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?query=flarn&id=userdoc:kittens:1234:foo")); - - assertEquals(1, factory.messages.size()); - assertEquals(2, result.hits().size()); - assertNotNull(result.hits().get("userdoc:kittens:5678:bar")); - assertNotNull(result.hits().get("userdoc:kittens:1234:foo")); - } - - @Test - public void testQueryPassThroughAndGetUnknownBackendHit() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - HitGroup hits = new HitGroup("mock"); - hits.add(new Hit("blernsball")); - Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(hits)); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?flarn=blern&id=userdoc:kittens:9:aaa")); - - assertEquals(0, factory.messages.size()); - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"18\" message=\"Internal server error.: " + - "A backend searcher to com.yahoo.storage.searcher.GetSearcher returned a " + - "hit that was not an instance of com.yahoo.storage.searcher.DocumentHit. " + - "Only DocumentHit instances are supported in the backend hit result set " + - "when doing queries that contain document identifier sets recognised by the " + - "Get Searcher.\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testConfig() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(58).route("route66").retryenabled(false)), defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=doc:batman:dahnahnahnah")); - - assertEquals(1, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("doc:batman:dahnahnahnah", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(Route.parse("route66"), gdm.getRoute()); - assertFalse(gdm.getRetryEnabled()); - assertTrue(58000 >= gdm.getTimeRemaining()); - } - } - - @Test - public void testConfigChanges() throws Exception { - String config = "raw:timeout 37\nroute \"riksveg18\"\nretryenabled true"; - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(58).route("riksveg18").retryenabled(true)), - defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=doc:batman:dahnahnahnah")); - - assertEquals(1, factory.messages.size()); - assertEquals(1, factory.getSessionsCreated()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("doc:batman:dahnahnahnah", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(Route.parse("riksveg18"), gdm.getRoute()); - assertTrue(gdm.getRetryEnabled()); - assertTrue(58000 >= gdm.getTimeRemaining()); - } - - factory.messages.clear(); - - FeederConfig newConfig = new FeederConfig(new FeederConfig.Builder() - .timeout(123) - .route("e6") - .retryenabled(false) - ); - searcher.getMessagePropertyProcessor().configure(newConfig, defLoadTypeCfg); - - new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=doc:spiderman:does_whatever_a_spider_can")); - - // riksveg18 is created again, and e6 is created as well. - assertEquals(3, factory.getSessionsCreated()); - - assertEquals(1, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("doc:spiderman:does_whatever_a_spider_can", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(Route.parse("e6"), gdm.getRoute()); - assertFalse(gdm.getRetryEnabled()); - assertTrue(123000 >= gdm.getTimeRemaining()); - } - } - - @Test - public void testQueryOverridesDefaults() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&priority=LOW_2&route=highwaytohell&timeout=58")); - - assertEquals(2, factory.messages.size()); - long lastTimeout = 58000; - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); - assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); - assertTrue(lastTimeout >= gdm.getTimeRemaining()); - lastTimeout = gdm.getTimeRemaining(); - } - - { - Message m = factory.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:3:4", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); - assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); - assertTrue(lastTimeout >= gdm.getTimeRemaining()); - } - } - - @Test - public void testQueryOverridesConfig() throws Exception { - String config = "raw:timeout 458\nroute \"route66\""; - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&priority=LOW_2&route=highwaytohell&timeout=123")); - - long lastTimeout = 123000; - assertEquals(2, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); - assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); - assertTrue(lastTimeout >= gdm.getTimeRemaining()); - lastTimeout = gdm.getTimeRemaining(); - } - - { - Message m = factory.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:3:4", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); - assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); - assertTrue(lastTimeout >= gdm.getTimeRemaining()); - } - } - - @Test - public void testBadPriorityValue() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:1:2&priority=onkel_jubalon")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: No enum const" + - "ant " + - "com.yahoo.documentapi.messagebus.protocol.DocumentProtocol" + - "." + - "Priority.onkel_jubalon\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testMultiIdBadArrayIndex() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - { - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[1]=userdoc:kittens:1:2")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: query contains document ID " + - "array that is not zero-based and/or linearly increasing\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - { - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[2]=userdoc:kittens:2:3")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: query contains document ID " + - "array that is not zero-based and/or linearly increasing\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - { - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[1]=userdoc:kittens:2:3")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: query contains document ID " + - "array that is not zero-based and/or linearly increasing\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - { - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0=userdoc:kittens:1:2")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: Malformed document ID array parameter\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - } - - @Test - public void testLegacyHeadersOnly() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:1:2&headersonly=true")); - - assertEquals(1, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[header]", gdm.getFieldSet()); - } - assertEquals(1, result.hits().size()); - assertHits(result.hits(), "userdoc:kittens:1:2"); - } - - @Test - public void testFieldSet() throws Exception { - } - - @Test - public void testConsistentResultOrdering() throws Exception { - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:1:2"))), - new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:7:8"))), - new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:555:123"))) - }; - - // Use a predefined reply list to ensure messages are answered out of order - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:555:123&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:7:8")); - - assertEquals(3, factory.messages.size()); - assertEquals(3, result.hits().size()); - // Hits must be in the same order as their document IDs in the query - assertHits(result.hits(), "userdoc:kittens:555:123", "userdoc:kittens:1:2", "userdoc:kittens:7:8"); - - assertEquals(0, ((DocumentHit)result.hits().get(0)).getIndex()); - assertEquals(1, ((DocumentHit)result.hits().get(1)).getIndex()); - assertEquals(2, ((DocumentHit)result.hits().get(2)).getIndex()); - } - - @Test - public void testResultWithSingleError() throws Exception { - com.yahoo.messagebus.Error err = new com.yahoo.messagebus.Error(32, "Alas, it went poorly"); - DocumentSessionFactory factory = new DocumentSessionFactory(docType, err, true); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4")); - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"messagebus\" code=\"32\" message=\"Alas, it went poorly\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testResultWithMultipleErrors() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); - Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:99:111")); - GetDocumentReply errorReply1 = new GetDocumentReply(doc1); - errorReply1.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 had fleas.")); - GetDocumentReply errorReply2 = new GetDocumentReply(doc2); - errorReply2.addError(new com.yahoo.messagebus.Error(456, "userdoc:kittens:99:111 shredded the curtains.")); - GetDocumentReply[] replies = new GetDocumentReply[] { - errorReply1, - errorReply2 - }; - - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:99:111")); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 had fleas.\"/>\n" + - "<error type=\"messagebus\" code=\"456\" message=\"userdoc:kittens:99:111 shredded the curtains.\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testResultWithNullDocument() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, true); - factory.setNullReply(true); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:55:bad_document_id")); - // Document not found does not produce any hit at all, error or otherwise - assertNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "</result>\n", result); - } - - @Test - public void testDefaultDocumentHitRendering() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:3:4")); - doc1.setFieldValue("name", new StringFieldValue("mittens")); - doc1.setFieldValue("description", new StringFieldValue("it's a cat")); - doc1.setFieldValue("fluffiness", new IntegerFieldValue(8)); - Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:1:2")); - doc2.setFieldValue("name", new StringFieldValue("garfield")); - doc2.setFieldValue("description", - new StringFieldValue("preliminary research indicates <em>hatred</em> of mondays. caution advised")); - doc2.setFieldValue("fluffiness", new IntegerFieldValue(2)); - Document doc3 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); - GetDocumentReply errorReply = new GetDocumentReply(doc3); - errorReply.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 had fleas.")); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - new GetDocumentReply(doc2), - errorReply - }; - - // Use a predefined reply list to ensure messages are answered out of order - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result xmlResult = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:3:4")); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 had fleas.\"/>\n" + - "</errors>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:1:2\">\n" + - " <name>garfield</name>\n" + - " <description>preliminary research indicates <em>hatred</em> of mondays. caution advised</description>\n" + - " <fluffiness>2</fluffiness>\n" + - "</document>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:3:4\">\n" + - " <name>mittens</name>\n" + - " <description>it's a cat</description>\n" + - " <fluffiness>8</fluffiness>\n" + - "</document>\n" + - "</result>\n", xmlResult); - } - - @Test - public void testDocumentFieldNoContentType() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - doc1.setFieldValue("name", "derrick"); - doc1.setFieldValue("description", "kommisar katze"); - doc1.setFieldValue("fluffiness", 0); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:5:1&field=description")); - - assertNull(result.hits().getErrorHit()); - assertEquals("text/xml", result.getTemplating().getTemplates().getMimeType()); - assertEquals("UTF-8", result.getTemplating().getTemplates().getEncoding()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>kommisar katze</result>\n", result); - } - - @Test - public void testDocumentFieldEscapeXML() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - doc1.setFieldValue("name", "asfd"); - doc1.setFieldValue("description", "<script type=\"evil/madness\">horror & screams</script>"); - doc1.setFieldValue("fluffiness", 0); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:5:1&field=description")); - - assertNull(result.hits().getErrorHit()); - assertEquals("text/xml", result.getTemplating().getTemplates().getMimeType()); - assertEquals("UTF-8", result.getTemplating().getTemplates().getEncoding()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result><script type=\"evil/madness\">horror & screams</script></result>\n", result); - } - - @Test - public void testDocumentFieldRawContent() throws Exception { - byte[] contentBytes = new byte[] { 0, -128, 127 }; - - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456")); - doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes))); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1) - }; - - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:123:456&field=foo")); - - assertNull(result.hits().getErrorHit()); - assertEquals("application/octet-stream", result.getTemplating().getTemplates().getMimeType()); - - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result); - stream.flush(); - - byte[] resultBytes = stream.toByteArray(); - assertEquals(contentBytes.length, resultBytes.length); - for (int i = 0; i < resultBytes.length; ++i) { - assertEquals(contentBytes[i], resultBytes[i]); - } - } - - @Test - public void testDocumentFieldRawWithContentOverride() throws Exception { - byte[] contentBytes = new byte[] { 0, -128, 127 }; - - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456")); - doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes))); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1) - }; - - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:123:456&field=foo&contenttype=text/fancy")); - - assertNull(result.hits().getErrorHit()); - assertEquals("text/fancy", result.getTemplating().getTemplates().getMimeType()); - - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result); - stream.flush(); - - byte[] resultBytes = stream.toByteArray(); - assertEquals(contentBytes.length, resultBytes.length); - for (int i = 0; i < resultBytes.length; ++i) { - assertEquals(contentBytes[i], resultBytes[i]); - } - } - - @Test - public void testDocumentFieldWithMultipleIDs() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&field=name")); - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: Field only valid for single document id query\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testDocumentFieldNotSet() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - doc1.setFieldValue("name", "asdf"); - doc1.setFieldValue("description", "fdsafsdf"); - doc1.setFieldValue("fluffiness", 10); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:5:1&field=image")); - - assertNotNull(result.hits().getErrorHit()); - assertEquals(1, result.hits().size()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"16\" message=\"Resource not found.: " + - "Field 'image' found in document type, but had no content in userdoc:kittens:5:1\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - - @Test - public void testDocumentFieldWithDocumentNotFound() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, true); - factory.setNullReply(true); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:1:2&field=name")); - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"16\" message=\"Resource not found.: " + - "Document not found, could not return field 'name'\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testDocumentFieldNotReachableWithHeadersOnly() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - doc1.setFieldValue("name", "asdf"); - // don't set body fields - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:5:1&field=description&headersonly=true")); - - assertNotNull(result.hits().getErrorHit()); - assertEquals(1, result.hits().size()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"4\" message=\"Invalid query parameter: " + - "Field 'description' is located in document body, but headersonly " + - "prevents it from being retrieved in userdoc:kittens:5:1\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testVespaXMLTemplate() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:3:4")); - doc1.setFieldValue("name", "mittens"); - doc1.setFieldValue("description", "it's a cat"); - doc1.setFieldValue("fluffiness", 8); - Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:1:2")); - doc2.setFieldValue("name", "garfield"); - doc2.setFieldValue("description", "preliminary research indicates <em>hatred</em> of mondays. caution advised"); - doc2.setFieldValue("fluffiness", 2); - Document doc3 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); - GetDocumentReply errorReply = new GetDocumentReply(doc3); - errorReply.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 lost in a <ni!>\"shrubbery\"</ni!>")); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - new GetDocumentReply(doc2), - errorReply - }; - - // Use a predefined reply list to ensure messages are answered out of order - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:3:4")); // TODO! - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 lost in a <ni!>"shrubbery"</ni!>\"/>\n"+ - "</errors>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:1:2\">\n" + - " <name>garfield</name>\n" + - " <description>preliminary research indicates <em>hatred</em> of mondays. caution advised</description>\n" + - " <fluffiness>2</fluffiness>\n" + - "</document>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:3:4\">\n" + - " <name>mittens</name>\n" + - " <description>it's a cat</description>\n" + - " <fluffiness>8</fluffiness>\n" + - "</document>\n" + - "</result>\n", result); - } - - @Test - public void testDocumentHitWithPopulatedHitFields() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:1234:foo")); - doc1.setFieldValue("name", new StringFieldValue("megacat")); - doc1.setFieldValue("description", new StringFieldValue("supercat")); - doc1.setFieldValue("fluffiness", new IntegerFieldValue(10000)); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1) - }; - - // Use a predefined reply list to ensure messages are answered out of order - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:1234:foo&populatehitfields=true")); - assertEquals(1, result.hits().size()); - assertHits(result.hits(), "userdoc:kittens:1234:foo"); - - DocumentHit hit = (DocumentHit)result.hits().get(0); - Iterator<Map.Entry<String, Object>> iter = hit.fieldIterator(); - Set<String> fieldSet = new TreeSet<>(); - while (iter.hasNext()) { - Map.Entry<String, Object> kv = iter.next(); - StringBuilder field = new StringBuilder(); - field.append(kv.getKey()).append(" -> ").append(kv.getValue()); - fieldSet.add(field.toString()); - } - StringBuilder fields = new StringBuilder(); - for (String s : fieldSet) { - fields.append(s).append("\n"); - } - assertEquals( - "description -> supercat\n" + - "documentid -> userdoc:kittens:1234:foo\n" + - "fluffiness -> 10000\n" + - "name -> megacat\n", - fields.toString()); - } - - @Test - public void deserializationExceptionsAreHandledGracefully() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - GetDocumentReply[] replies = new GetDocumentReply[] { - new MockFailingGetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:5:1")); - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"18\" message=\"Internal server error.: " + - "Got exception of type java.lang.RuntimeException during document " + - "deserialization: epic dragon attack\"/>\n"+ - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testJsonRendererSetting() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Query query = newQuery("?id=userdoc:kittens:1:2&format=json"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - assertFalse(result.getTemplating().getTemplates() instanceof DocumentXMLTemplate); - } - - private Query newQuery(String queryString) { - return new Query(HttpRequest.createTestRequest(queryString, com.yahoo.jdisc.http.HttpRequest.Method.GET)); - } - - private Chain<Searcher> createSearcherChain(GetDocumentReply[] replies) throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - return new Chain<>(searcher); - } - - private static class MockFailingGetDocumentReply extends GetDocumentReply { - private int countdown = 2; - - private MockFailingGetDocumentReply(Document doc) { - super(doc); - } - - @Override - public Document getDocument() { - // Reason for countdown is that the test DocumentSessionFactory calls - // getDocument once internally before code can ever reach handleReply. - if (--countdown == 0) { - throw new RuntimeException("epic dragon attack"); - } - return super.getDocument(); - } - } - - private static class MockBackend extends Searcher { - private Hit hitToReturn; - - public MockBackend(Hit hitToReturn) { - this.hitToReturn = hitToReturn; - } - - @Override - public Result search(Query query, Execution execution) { - Result result = new Result(query); - result.hits().add(hitToReturn); - return result; - } - } - - private class MockHttpRequest { - - private final String req; - private byte[] data; - private boolean gzip = false; - - MockHttpRequest(byte[] data, String req) { - this.req = req; - this.data = data; - } - - MockHttpRequest(byte[] data, String req, boolean gzip) { - this.data = data; - this.req = req; - this.gzip = gzip; - } - - public InputStream getData() { - if (gzip) { - try { - ByteArrayOutputStream rawOut = new ByteArrayOutputStream(); - GZIPOutputStream compressed = new GZIPOutputStream(rawOut); - compressed.write(data, 0, data.length); - compressed.finish(); - compressed.flush(); - rawOut.flush(); - return new ByteArrayInputStream(rawOut.toByteArray()); - } catch (Exception e) { - return null; - } - } - return new ByteArrayInputStream(data); - } - - public void addHeaders(HeaderFields headers) { - headers.add("Content-Type", "text/plain;encoding=UTF-8"); - if (gzip) - headers.add("Content-Encoding", "gzip"); - } - - public com.yahoo.container.jdisc.HttpRequest toRequest() { - com.yahoo.container.jdisc.HttpRequest request = com.yahoo.container.jdisc.HttpRequest.createTestRequest(req, com.yahoo.jdisc.http.HttpRequest.Method.GET, getData()); - addHeaders(request.getJDiscRequest().headers()); - return request; - } - - } - - public static void assertRendered(String expected,Result result) throws Exception { - assertRendered(expected,result,true); - } - - public static void assertRendered(String expected,Result result,boolean checkFullEquality) throws Exception { - if (checkFullEquality) - assertEquals(expected, ResultRenderingUtil.getRendered(result)); - else - assertTrue(ResultRenderingUtil.getRendered(result).startsWith(expected)); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java deleted file mode 100644 index 7f392b0565b..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.search.Result; - -import java.io.ByteArrayOutputStream; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; - -@SuppressWarnings("deprecation") -public class ResultRenderingUtil { - - public static String getRendered(Result result) throws Exception { - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - Charset cs = Charset.forName("utf-8"); - CharsetDecoder decoder = cs.newDecoder(); - com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result); - stream.flush(); - return decoder.decode(ByteBuffer.wrap(stream.toByteArray())).toString(); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java deleted file mode 100644 index e49d2ca4db3..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.component.chain.Chain; -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.protect.Error; -import com.yahoo.documentapi.VisitorSession; -import com.yahoo.feedhandler.NullFeedMetric; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.document.DataType; -import com.yahoo.document.DocumentType; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.documentapi.VisitorParameters; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.messagebus.StaticThrottlePolicy; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.vdslib.VisitorOrdering; -import com.yahoo.vespaclient.ClusterList; -import com.yahoo.vespaclient.config.FeederConfig; - -import org.junit.Test; - -import java.util.Arrays; - -import static org.junit.Assert.*; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@SuppressWarnings("deprecation") -public class VisitorSearcherTestCase { - - private DocumentTypeManager docMan = null; - private DocumentType docType; - DocumentSessionFactory factory; - - @org.junit.Before - public void setUp() { - docMan = new DocumentTypeManager(); - docType = new DocumentType("kittens"); - docType.addHeaderField("name", DataType.STRING); - docType.addField("description", DataType.STRING); - docType.addField("image", DataType.RAW); - docType.addField("fluffiness", DataType.INT); - docType.addField("foo", DataType.RAW); - docMan.registerDocumentType(docType); - factory = new DocumentSessionFactory(docType); - } - - public VisitSearcher create() throws Exception { - ClusterListConfig.Storage.Builder storageCluster = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foobar").name("foobar"); - ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(storageCluster)); - ClusterList clusterList = new ClusterList(clusterListCfg); - return new VisitSearcher(new FeedContext( - new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(458).route("riksveg18").retryenabled(true)), - new LoadTypeConfig(new LoadTypeConfig.Builder())), - factory, docMan, clusterList, new NullFeedMetric(true))); - } - - @Test - public void testQueryParameters() throws Exception { - VisitSearcher searcher = create(); - VisitorParameters params = searcher.getVisitorParameters( - newQuery("visit?visit.selection=id.user=1234&visit.cluster=foobar" + - "&visit.dataHandler=othercluster&visit.fieldSet=[header]&visit.fromTimestamp=112&visit.toTimestamp=224" + - "&visit.maxBucketsPerVisitor=2&visit.maxPendingMessagesPerVisitor=7&visit.maxPendingVisitors=14" + - "&visit.ordering=ASCENDING&priority=NORMAL_1&tracelevel=7&visit.visitInconsistentBuckets&visit.visitRemoves"), null); - - assertEquals("id.user=1234", params.getDocumentSelection()); - assertEquals(7, params.getMaxPending()); - assertEquals(2, params.getMaxBucketsPerVisitor()); - assertEquals(14, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount()); - assertEquals("[Storage:cluster=foobar;clusterconfigid=storage/cluster.foobar]", params.getRoute().toString()); - assertEquals("othercluster", params.getRemoteDataHandler()); - assertEquals("[header]", params.fieldSet()); - assertEquals(112, params.getFromTimestamp()); - assertEquals(224, params.getToTimestamp()); - assertEquals(VisitorOrdering.ASCENDING, params.getVisitorOrdering()); - assertEquals(DocumentProtocol.Priority.NORMAL_1, params.getPriority()); - assertEquals(7, params.getTraceLevel()); - assertEquals(true, params.visitInconsistentBuckets()); - assertEquals(true, params.visitRemoves()); - } - - @Test - public void timestampQueryParametersAreParsedAsLongs() throws Exception { - VisitorParameters params = create().getVisitorParameters( - newQuery("visit?visit.selection=id.user=1234&" + - "visit.fromTimestamp=1419021596000000&" + - "visit.toTimestamp=1419021597000000"), null); - assertEquals(1419021596000000L, params.getFromTimestamp()); - assertEquals(1419021597000000L, params.getToTimestamp()); - } - - @Test - public void testQueryParametersDefaults() throws Exception { - VisitSearcher searcher = create(); - VisitorParameters params = searcher.getVisitorParameters( - newQuery("visit?visit.selection=id.user=1234&hits=100"), null); - - assertEquals("id.user=1234", params.getDocumentSelection()); - assertEquals(1, params.getMaxBucketsPerVisitor()); - assertEquals(1, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount()); - assertEquals(1, params.getMaxFirstPassHits()); - assertEquals(1, params.getMaxTotalHits()); - assertEquals(32, params.getMaxPending()); - assertEquals(false, params.visitInconsistentBuckets()); - } - - @Test - public void testWrongCluster() throws Exception { - VisitSearcher searcher = create(); - - try { - searcher.getVisitorParameters( - newQuery("visit?visit.selection=id.user=1234&visit.cluster=unknown"), null); - - assertTrue(false); - } catch (Exception e) { - // e.printStackTrace(); - } - } - - - @Test(expected = IllegalArgumentException.class) - public void testNoClusterParamWhenSeveralClusters() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - ClusterListConfig.Storage.Builder storageCluster1 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foo").name("foo"); - ClusterListConfig.Storage.Builder storageCluster2 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.bar").name("bar"); - ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(Arrays.asList(storageCluster1, storageCluster2))); - ClusterList clusterList = new ClusterList(clusterListCfg); - VisitSearcher searcher = new VisitSearcher(new FeedContext( - new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(100).route("whatever").retryenabled(true)), - new LoadTypeConfig(new LoadTypeConfig.Builder())), - factory, docMan, clusterList, new NullFeedMetric(true))); - - searcher.getVisitorParameters(newQuery("visit?visit.selection=id.user=1234"), null); - } - - @Test - public void testSimple() throws Exception { - Chain<Searcher> searchChain = new Chain<>(create()); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("visit?visit.selection=id.user=1234&hits=100")); - assertEquals(1, result.hits().size()); - assertRendered( - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:foo:1234:bar\"/>\n" + - "</result>\n", result); - } - - private Result invokeVisitRemovesSearchChain() throws Exception { - Chain<Searcher> searchChain = new Chain<>(create()); - return new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("visit?visit.selection=id.user=1234&hits=100&visit.visitRemoves=true")); - } - - @Test - public void visitRemovesIncludesRemoveEntriesInResultXml() throws Exception { - Result result = invokeVisitRemovesSearchChain(); - assertEquals(2, result.hits().size()); - assertRendered( - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:foo:1234:bar\"/>\n" + - "<remove documentid=\"userdoc:foo:1234:removed\"/>\n" + - "</result>\n", result); - } - - @Test - public void removedDocumentIdsAreXmlEscaped() throws Exception { - factory = mock(DocumentSessionFactory.class); - when(factory.createVisitorSession(any(VisitorParameters.class))).thenAnswer((p) -> { - VisitorParameters params = (VisitorParameters)p.getArguments()[0]; - DummyVisitorSession session = new DummyVisitorSession(params, docType); - session.clearAutoReplyMessages(); - session.addRemoveReply("userdoc:foo:1234:<rem\"o\"ved&stuff>"); - return session; - }); - Result result = invokeVisitRemovesSearchChain(); - assertEquals(1, result.hits().size()); - assertRendered( - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<remove documentid=\"userdoc:foo:1234:<rem"o"ved&stuff>\"/>\n" + - "</result>\n", result); - } - - private Result invokeSearcherWithUserQuery() throws Exception { - Chain<Searcher> searchChain = new Chain<>(create()); - return new Execution(searchChain, Execution.Context.createContextStub()) - .search(new Query("visit?visit.selection=id.user=1234&hits=100")); - } - - @Test - public void waitUntilDoneFailureReturnsTimeoutErrorHit() throws Exception { - VisitorSession session = mock(VisitorSession.class); - when(session.waitUntilDone(anyLong())).thenReturn(false); - factory = mock(DocumentSessionFactory.class); - when(factory.createVisitorSession(any(VisitorParameters.class))).thenReturn(session); - - Result result = invokeSearcherWithUserQuery(); - assertNotNull(result.hits().getErrorHit()); - assertEquals(Error.TIMEOUT.code, result.hits().getErrorHit().errors().iterator().next().getCode()); - } - - @Test - @SuppressWarnings("deprecation") - public void testRendererWiring() throws Exception { - Chain<Searcher> searchChain = new Chain<>(create()); - { - Query query = newQuery("visit?visit.selection=id.user=1234&hits=100&format=json"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - assertEquals(com.yahoo.prelude.templates.DefaultTemplateSet.class, result.getTemplating().getTemplates().getClass()); - } - { - Query query = newQuery("visit?visit.selection=id.user=1234&hits=100&format=JsonRenderer"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - assertEquals(com.yahoo.prelude.templates.DefaultTemplateSet.class, result.getTemplating().getTemplates().getClass()); - } - { - Query query = newQuery("visit?visit.selection=id.user=1234&hits=100"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - assertEquals(DocumentXMLTemplate.class, result.getTemplating().getTemplates().getClass()); - } - } - - public static void assertRendered(String expected, Result result) throws Exception { - assertEquals(expected, ResultRenderingUtil.getRendered(result)); - } - - private Query newQuery(String queryString) { - return new Query(HttpRequest.createTestRequest(queryString, com.yahoo.jdisc.http.HttpRequest.Method.GET)); - } - -} |