diff options
Diffstat (limited to 'vespaclient-java/src/test/java')
10 files changed, 1783 insertions, 0 deletions
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java new file mode 100644 index 00000000000..2b0b4cc9048 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java @@ -0,0 +1,77 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.concurrent.Timer; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.messagebus.EmptyReply; +import junit.framework.TestCase; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +/** + */ +public class BenchmarkProgressPrinterTest extends TestCase { + + class DummyTimer implements Timer { + long ms; + + public long milliTime() { return ms; } + } + + public void testSimple() { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + DummyTimer timer = new DummyTimer(); + timer.ms = 0; + BenchmarkProgressPrinter printer = new BenchmarkProgressPrinter(timer, new PrintStream(output)); + RouteMetricSet metrics = new RouteMetricSet("foobar", printer); + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(1)); + metrics.addReply(reply); + } + + timer.ms = 1200; + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(2)); + metrics.addReply(reply); + } + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(3)); + metrics.addReply(reply); + } + + timer.ms = 2400; + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(4)); + reply.addError(new com.yahoo.messagebus.Error(32, "foo")); + metrics.addReply(reply); + } + + timer.ms = 62000; + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(5)); + reply.addError(new com.yahoo.messagebus.Error(64, "bar")); + metrics.addReply(reply); + } + + metrics.done(); + + String val = output.toString().split("\n")[1]; + + String correctPattern = "62000,\\s*3,\\s*2,\\s*\\d+,\\s*\\d+,\\s*\\d+$"; + assertTrue("Value '" + val + "' does not match pattern '" + correctPattern + "'", val.matches(correctPattern)); + } + +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java new file mode 100644 index 00000000000..ae49bb1318d --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java @@ -0,0 +1,90 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.concurrent.Timer; +import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.messagebus.EmptyReply; +import junit.framework.TestCase; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +/** + */ +public class ProgressPrinterTest extends TestCase { + + class DummyTimer implements Timer { + long ms; + + public long milliTime() { return ms; } + } + + public void testSimple() { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + DummyTimer timer = new DummyTimer(); + timer.ms = 0; + ProgressPrinter printer = new ProgressPrinter(timer, new PrintStream(output)); + RouteMetricSet metrics = new RouteMetricSet("foobar", printer); + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(1)); + metrics.addReply(reply); + } + + timer.ms = 1200; + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(2)); + metrics.addReply(reply); + } + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(3)); + metrics.addReply(reply); + } + + timer.ms = 2400; + + { + DocumentIgnoredReply reply = new DocumentIgnoredReply(); + reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(0)); + metrics.addReply(reply); + } + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(5)); + reply.addError(new com.yahoo.messagebus.Error(32, "foo")); + metrics.addReply(reply); + } + + timer.ms = 62000; + + { + EmptyReply reply = new EmptyReply(); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(6)); + reply.addError(new com.yahoo.messagebus.Error(64, "bar")); + metrics.addReply(reply); + } + + String val = output.toString().replaceAll("latency\\(min, max, avg\\): .*", "latency(min, max, avg): 0, 0, 0"); + + String correct = + "\rSuccessfully sent 2 messages so far" + + "\rSuccessfully sent 3 messages so far" + + "\n\n" + + "Messages sent to vespa (route foobar) :\n" + + "---------------------------------------\n" + + "PutDocument:\tok: 2 msgs/sec: 0.03 failed: 0 ignored: 1 latency(min, max, avg): 0, 0, 0\n" + + "UpdateDocument:\tok: 1 msgs/sec: 0.02 failed: 2 ignored: 0 latency(min, max, avg): 0, 0, 0\n"; + + assertEquals(correct, val); + } + +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java new file mode 100644 index 00000000000..42d4b082ff3 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java @@ -0,0 +1,208 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import static org.junit.Assert.*; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Arrays; + +import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.feedapi.DummySessionFactory; +import com.yahoo.feedhandler.VespaFeedHandler; +import com.yahoo.text.Utf8; +import com.yahoo.vespaclient.config.FeederConfig; +import com.yahoo.vespafeeder.Arguments.HelpShownException; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class VespaFeederTestCase { + + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Test + public void testParseArgs() throws Exception { + String argsS="--abortondataerror false --abortonsenderror false --file foo.xml --maxpending 10" + + " --maxpendingsize 11 --maxfeedrate 29 --mode benchmark --noretry --retrydelay 12 --route e6 --timeout 13 --trace 4" + + " --validate -v bar.xml --priority LOW_1"; + + Arguments arguments = new Arguments(argsS.split(" "), DummySessionFactory.createWithAutoReply()); + + FeederConfig config = arguments.getFeederConfig(); + assertEquals(false, config.abortondocumenterror()); + assertEquals(13.0, config.timeout(), 0.00001); + assertEquals(false, config.retryenabled()); + assertEquals(12.0, config.retrydelay(), 0.0001); + assertEquals("e6", config.route()); + assertEquals(4, config.tracelevel()); + assertEquals(false, config.abortonsenderror()); + assertEquals(10, config.maxpendingdocs()); + assertEquals(11, config.maxpendingbytes()); + assertEquals(29.0, config.maxfeedrate(), 0.0001); + assertTrue(arguments.isVerbose()); + assertFalse(config.createifnonexistent()); + + assertEquals("LOW_1", arguments.getPriority()); + assertEquals("benchmark", arguments.getMode()); + assertEquals("foo.xml", arguments.getFiles().get(0)); + assertEquals("bar.xml", arguments.getFiles().get(1)); + } + + @Test + public void requireThatCreateIfNonExistentArgumentCanBeParsed() throws Exception { + String argsS="--create-if-non-existent --file foo.xml"; + Arguments arguments = new Arguments(argsS.split(" "), DummySessionFactory.createWithAutoReply()); + assertTrue(arguments.getFeederConfig().createifnonexistent()); + } + + @Test + public void testHelp() throws Exception { + String argsS="-h"; + + try { + new Arguments(argsS.split(" "), null); + assertTrue(false); + } catch (Arguments.HelpShownException e) { + + } + } + + @Test + public void requireCorrectInputTypeDetection() throws IOException { + { + BufferedInputStream b = new BufferedInputStream( + new ByteArrayInputStream(Utf8.toBytes("[]"))); + InputStreamRequest r = new InputStreamRequest(b); + VespaFeeder.setJsonInput(r, b); + assertEquals("true", r.getProperty(VespaFeedHandler.JSON_INPUT)); + } + { + BufferedInputStream b = new BufferedInputStream( + new ByteArrayInputStream(Utf8.toBytes("<document />"))); + InputStreamRequest r = new InputStreamRequest(b); + VespaFeeder.setJsonInput(r, b); + assertEquals("false", r.getProperty(VespaFeedHandler.JSON_INPUT)); + } + } + + public void assertRenderErrorOutput(String expected, String[] errors) { + ArrayList<String> l = new ArrayList<String>(); + l.addAll(Arrays.asList(errors)); + assertEquals(expected, VespaFeeder.renderErrors(l).getMessage()); + } + + @Test + public void testRenderErrors() { + { + String[] errors = { "foo" }; + assertRenderErrorOutput("Errors:\n" + + "-------\n" + + " foo\n", errors); + } + + { + String[] errors = { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"}; + assertRenderErrorOutput("First 10 errors (of 11):\n" + + "------------------------\n" + + " 1\n 2\n 3\n 4\n 5\n 6\n 7\n 8\n 9\n 10\n", errors); + } + } + + public RouteMetricSet.ProgressCallback getProgressPrinter(String args) throws Exception { + Arguments arguments = new Arguments(args.split(" "), DummySessionFactory.createWithAutoReply()); + return new VespaFeeder(arguments, null).createProgressCallback(System.out); + } + + @Test + public void testCreateProgressPrinter() throws Exception { + assert(getProgressPrinter("--mode benchmark") instanceof BenchmarkProgressPrinter); + assert(getProgressPrinter("") instanceof ProgressPrinter); + } + + private static class FeedFixture { + DummySessionFactory sessionFactory = DummySessionFactory.createWithAutoReply(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + PrintStream printStream = new PrintStream(outputStream); + DocumentTypeManager typeManager = new DocumentTypeManager(); + FeedFixture() { + DocumentTypeManagerConfigurer.configure(typeManager, "file:src/test/files/documentmanager.cfg"); + } + } + + @Test + public void feedFile() throws Exception { + FeedFixture f = new FeedFixture(); + Arguments arguments = new Arguments("--file src/test/files/myfeed.xml --priority LOW_1".split(" "), f.sessionFactory); + new VespaFeeder(arguments, f.typeManager).parseFiles(System.in, f.printStream); + + assertEquals(3, f.sessionFactory.messages.size()); + assertEquals(DocumentProtocol.Priority.LOW_1, ((PutDocumentMessage)f.sessionFactory.messages.get(0)).getPriority()); + assertEquals("doc:test:foo", ((PutDocumentMessage) f.sessionFactory.messages.get(0)).getDocumentPut().getDocument().getId().toString()); + DocumentUpdate update = ((UpdateDocumentMessage) f.sessionFactory.messages.get(1)).getDocumentUpdate(); + assertEquals("doc:test:foo", update.getId().toString()); + assertFalse(update.getCreateIfNonExistent()); + assertEquals("doc:test:foo", ((RemoveDocumentMessage) f.sessionFactory.messages.get(2)).getDocumentId().toString()); + + assertTrue(f.outputStream.toString().contains("Messages sent to vespa")); + } + + @Test + public void feedJson() throws Exception { + FeedFixture feedFixture = feed("src/test/files/myfeed.json", true); + + assertJsonFeedState(feedFixture); + } + + protected void assertJsonFeedState(FeedFixture feedFixture) { + assertEquals(3, feedFixture.sessionFactory.messages.size()); + assertEquals(DocumentProtocol.Priority.LOW_1, ((PutDocumentMessage)feedFixture.sessionFactory.messages.get(0)).getPriority()); + assertEquals("id:test:news::foo", ((PutDocumentMessage) feedFixture.sessionFactory.messages.get(0)).getDocumentPut().getDocument().getId().toString()); + DocumentUpdate update = ((UpdateDocumentMessage) feedFixture.sessionFactory.messages.get(1)).getDocumentUpdate(); + assertEquals("id:test:news::foo", update.getId().toString()); + assertFalse(update.getCreateIfNonExistent()); + assertEquals("id:test:news::foo", ((RemoveDocumentMessage) feedFixture.sessionFactory.messages.get(2)).getDocumentId().toString()); + + assertTrue(feedFixture.outputStream.toString().contains("Messages sent to vespa")); + } + + @Test + public void requireThatCreateIfNonExistentArgumentIsUsed() throws Exception { + FeedFixture f = new FeedFixture(); + Arguments arguments = new Arguments("--file src/test/files/myfeed.xml --create-if-non-existent".split(" "), f.sessionFactory); + new VespaFeeder(arguments, f.typeManager).parseFiles(System.in, f.printStream); + + assertEquals(3, f.sessionFactory.messages.size()); + DocumentUpdate update = ((UpdateDocumentMessage) f.sessionFactory.messages.get(1)).getDocumentUpdate(); + assertTrue(update.getCreateIfNonExistent()); + } + + @Test + public void feedMalformedJson() throws Exception { + exception.expect(VespaFeeder.FeedErrorException.class); + exception.expectMessage("JsonParseException"); + feed("src/test/files/malformedfeed.json", false); + } + + protected FeedFixture feed(String feed, boolean abortOnDataError) throws HelpShownException, + FileNotFoundException, Exception { + String abortOnDataErrorArgument = abortOnDataError ? "" : " --abortondataerror no"; + FeedFixture feedFixture = new FeedFixture(); + Arguments arguments = new Arguments(("--file " + + feed + + " --priority LOW_1" + abortOnDataErrorArgument).split(" "), feedFixture.sessionFactory); + new VespaFeeder(arguments, feedFixture.typeManager).parseFiles(System.in, feedFixture.printStream); + return feedFixture; + } +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java new file mode 100644 index 00000000000..3e707b04256 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java @@ -0,0 +1,195 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.*; +import java.util.Iterator; + +import static org.junit.Assert.*; + +/** + * Test class for {@link CommandLineOptions} + * + * @author bjorncs + * @since 5.26 + */ +public class CommandLineOptionsTest { + + private final InputStream emptyStream = new InputStream() { + + @Override + public int read() throws IOException { + return -1; + } + }; + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + private ClientParameters getParsedOptions(InputStream in, String... args) { + CommandLineOptions options = new CommandLineOptions(in); + return options.parseCommandLineArguments(args); + } + + private ClientParameters getParsedOptions(String... args) { + return getParsedOptions(emptyStream, args); + } + + @Test + public void testDefaultOptions() { + ClientParameters params = getParsedOptions(); + assertFalse(params.help); + assertFalse(params.documentIds.hasNext()); + assertFalse(params.printIdsOnly); + assertEquals("[all]", params.fieldSet); + assertEquals("default", params.route); + assertTrue(params.cluster.isEmpty()); + assertEquals("client", params.configId); + assertFalse(params.showDocSize); + assertEquals(0, params.timeout, 0); + assertFalse(params.noRetry); + assertEquals(0, params.traceLevel); + assertEquals(DocumentProtocol.Priority.NORMAL_2, params.priority); + assertTrue(params.loadTypeName.isEmpty()); + } + + @Test + public void testValidOptions() { + ClientParameters params = getParsedOptions( + "--fieldset", "[fieldset]", + "--route", "dummyroute", + "--configid", "dummyconfig", + "--showdocsize", + "--timeout", "0.25", + "--noretry", + "--trace", "1", + "--priority", Integer.toString(DocumentProtocol.Priority.HIGH_3.getValue()), + "--loadtype", "dummyloadtype", + "id:1", "id:2" + ); + + assertEquals("[fieldset]", params.fieldSet); + assertEquals("dummyroute", params.route); + assertEquals("dummyconfig", params.configId); + assertTrue(params.showDocSize); + assertEquals(0.25, params.timeout, 0.0001); + assertTrue(params.noRetry); + assertEquals(1, params.traceLevel); + assertEquals(DocumentProtocol.Priority.HIGH_3, params.priority); + assertEquals("dummyloadtype", params.loadTypeName); + + Iterator<String> documentsIds = params.documentIds; + assertEquals("id:1", documentsIds.next()); + assertEquals("id:2", documentsIds.next()); + assertFalse(documentsIds.hasNext()); + } + + @Test + public void testInvalidCombination1() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Print ids and headers only options are mutually exclusive."); + getParsedOptions("--headersonly", "--printids"); + } + + @Test + public void testInvalidCombination2() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Field set option can not be used in combination with print ids or headers only options."); + getParsedOptions("--headersonly", "--fieldset", "[header]"); + } + + @Test + public void testInvalidCombination3() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Field set option can not be used in combination with print ids or headers only options."); + getParsedOptions("--printids", "--fieldset", "[header]"); + } + + @Test + public void testInvalidCombination4() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Cluster and route options are mutually exclusive."); + getParsedOptions("--route", "dummyroute", "--cluster", "dummycluster"); + } + + @Test + public void testInvalidPriority() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Invalid priority: 16"); + getParsedOptions("--priority", "16"); + } + + @Test + public void testInvalidTraceLevel1() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Invalid tracelevel: -1"); + getParsedOptions("--trace", "-1"); + } + + @Test + public void testInvalidTraceLevel2() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Invalid tracelevel: 10"); + getParsedOptions("--trace", "10"); + } + + @Test + public void testPrintids() { + ClientParameters params = getParsedOptions("--printids"); + assertEquals("[id]", params.fieldSet); + } + + @Test + public void testHeadersOnly() { + ClientParameters params = getParsedOptions("--headersonly"); + assertEquals("[header]", params.fieldSet); + } + + @Test + public void testCluster() { + ClientParameters params = getParsedOptions("--cluster", "dummycluster"); + assertEquals("dummycluster", params.cluster); + assertTrue(params.route.isEmpty()); + } + + @Test + public void testHelp() { + ClientParameters params = getParsedOptions("--help"); + assertTrue(params.help); + } + + @Test + public void testDocumentIdsFromInputStream() throws UnsupportedEncodingException { + InputStream in = new ByteArrayInputStream("id:1 id:2 id:3".getBytes("UTF-8")); + ClientParameters params = getParsedOptions(in, ""); + + Iterator<String> documentsIds = params.documentIds; + assertEquals("id:1", documentsIds.next()); + assertEquals("id:2", documentsIds.next()); + assertEquals("id:3", documentsIds.next()); + assertFalse(documentsIds.hasNext()); + } + + @Test + public void testPrintHelp() { + ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + PrintStream oldOut = System.out; + System.setOut(new PrintStream(outContent)); + try { + CommandLineOptions options = new CommandLineOptions(emptyStream); + options.printHelp(); + + String output = outContent.toString(); + assertTrue(output.contains("vespaget <options> [documentid...]")); + assertTrue(output.contains("Fetch a document from a Vespa Content cluster.")); + } finally { + System.setOut(oldOut); + outContent.reset(); + + } + } +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java new file mode 100644 index 00000000000..c3d3fcc71e9 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java @@ -0,0 +1,376 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.MessageBusSyncSession; +import com.yahoo.documentapi.messagebus.loadtypes.LoadType; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.Reply; +import com.yahoo.vespaclient.ClusterDef; +import com.yahoo.vespaclient.ClusterList; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.mockito.ArgumentMatcher; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Test class for {@link DocumentRetriever} + * + * @author bjorncs + */ +public class DocumentRetrieverTest { + + public static final String DOC_ID_1 = "id:storage_test:document::1"; + public static final String DOC_ID_2 = "id:storage_test:document::2"; + public static final String DOC_ID_3 = "id:storage_test:document::3"; + + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + + private DocumentAccessFactory mockedFactory; + private MessageBusDocumentAccess mockedDocumentAccess; + private MessageBusSyncSession mockedSession; + private PrintStream oldOut; + private PrintStream oldErr; + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Before + public void setUpStreams() { + oldOut = System.out; + oldErr = System.err; + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @Before + public void prepareMessageBusMocks() { + this.mockedFactory = mock(DocumentAccessFactory.class); + this.mockedDocumentAccess = mock(MessageBusDocumentAccess.class); + this.mockedSession = mock(MessageBusSyncSession.class); + when(mockedFactory.createDocumentAccess(any())).thenReturn(mockedDocumentAccess); + when(mockedDocumentAccess.createSyncSession(any())).thenReturn(mockedSession); + } + + @After + public void cleanUpStreams() { + System.setOut(oldOut); + System.setErr(oldErr); + outContent.reset(); + errContent.reset(); + } + + private static ClientParameters.Builder createParameters() { + return new ClientParameters.Builder() + .setPriority(DocumentProtocol.Priority.NORMAL_2) + .setCluster("") + .setRoute("default") + .setConfigId("client") + .setFieldSet("[all]") + .setPrintIdsOnly(false) + .setHelp(false) + .setShowDocSize(false) + .setLoadTypeName("") + .setNoRetry(false) + .setTraceLevel(0) + .setTimeout(0) + .setDocumentIds(Collections.emptyIterator()); + } + + private static Iterator<String> asIterator(String... docIds) { + return Arrays.asList(docIds).iterator(); + } + + private static Reply createDocumentReply(String docId) { + return new GetDocumentReply(new Document(DataType.DOCUMENT, new DocumentId(docId))); + } + + private void assertContainsDocument(String documentId) { + assertTrue(outContent.toString().contains(String.format( + "<document documenttype=\"document\" documentid=\"%s\"/>", documentId))); + } + + private DocumentRetriever createDocumentRetriever(ClientParameters params) { + return createDocumentRetriever(params, new ClusterList()); + } + + private DocumentRetriever createDocumentRetriever(ClientParameters params, ClusterList clusterList) { + return new DocumentRetriever( + clusterList, + mockedFactory, + new LoadTypeSet(), + params); + } + + @Test + public void testSendSingleMessage() throws DocumentRetrieverException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1)) + .setPriority(DocumentProtocol.Priority.HIGH_1) + .setNoRetry(true) + .setLoadTypeName("loadtype") + .build(); + + when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1)); + + LoadTypeSet loadTypeSet = new LoadTypeSet(); + loadTypeSet.addLoadType(1, "loadtype", DocumentProtocol.Priority.HIGH_1); + DocumentRetriever documentRetriever = new DocumentRetriever( + new ClusterList(), + mockedFactory, + loadTypeSet, + params); + documentRetriever.retrieveDocuments(); + + verify(mockedSession, times(1)).syncSend(argThat(new ArgumentMatcher<GetDocumentMessage>() { + @Override + public boolean matches(Object o) { + GetDocumentMessage msg = (GetDocumentMessage) o; + return msg.getPriority().equals(DocumentProtocol.Priority.HIGH_1) && + !msg.getRetryEnabled() && + msg.getLoadType().equals(new LoadType(1, "loadtype", DocumentProtocol.Priority.HIGH_1)); + } + })); + assertContainsDocument(DOC_ID_1); + } + + @Test + public void testMultipleMessages() throws DocumentRetrieverException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1, DOC_ID_2, DOC_ID_3)) + .build(); + + when(mockedSession.syncSend(any())).thenReturn( + createDocumentReply(DOC_ID_1), + createDocumentReply(DOC_ID_2), + createDocumentReply(DOC_ID_3)); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + + verify(mockedSession, times(3)).syncSend(any()); + assertContainsDocument(DOC_ID_1); + assertContainsDocument(DOC_ID_2); + assertContainsDocument(DOC_ID_3); + } + + @Test + public void testJsonOutput() throws DocumentRetrieverException, JsonParseException, IOException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1, DOC_ID_2, DOC_ID_3)) + .setJsonOutput(true) + .build(); + + when(mockedSession.syncSend(any())).thenReturn( + createDocumentReply(DOC_ID_1), + createDocumentReply(DOC_ID_2), + createDocumentReply(DOC_ID_3)); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + + verify(mockedSession, times(3)).syncSend(any()); + ObjectMapper m = new ObjectMapper(); + @SuppressWarnings("unchecked") + List<Map<String, Object>> feed = m.readValue(outContent.toByteArray(), List.class); + assertEquals(DOC_ID_1, feed.get(0).get("id")); + assertEquals(DOC_ID_2, feed.get(1).get("id")); + assertEquals(DOC_ID_3, feed.get(2).get("id")); + } + + @Test + public void testShutdownHook() throws DocumentRetrieverException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1)) + .build(); + + when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1)); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + documentRetriever.shutdown(); + + verify(mockedSession, times(1)).destroy(); + verify(mockedDocumentAccess, times(1)).shutdown(); + } + + @Test + public void testInvalidLoadType() throws DocumentRetrieverException { + exception.expect(DocumentRetrieverException.class); + exception.expectMessage("Loadtype with name 'undefinedloadtype' does not exist.\n"); + + ClientParameters params = createParameters() + .setLoadTypeName("undefinedloadtype") + .build(); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + } + + @Test + public void testClusterLookup() throws DocumentRetrieverException { + final String cluster = "storage", configId = "content/cluster.foo/storage", + expectedRoute = "[Storage:cluster=storage;clusterconfigid=content/cluster.foo/storage]"; + + ClientParameters params = createParameters() + .setCluster(cluster) + .build(); + + ClusterList clusterList = new ClusterList(Collections.singletonList(new ClusterDef(cluster, configId))); + + DocumentRetriever documentRetriever = createDocumentRetriever(params, clusterList); + documentRetriever.retrieveDocuments(); + + verify(mockedFactory).createDocumentAccess(argThat(new ArgumentMatcher<MessageBusParams>() { + @Override + public boolean matches(Object o) { + return ((MessageBusParams) o).getRoute().equals(expectedRoute); + } + })); + } + + @Test + public void testInvalidClusterName() throws DocumentRetrieverException { + exception.expect(DocumentRetrieverException.class); + exception.expectMessage("The Vespa cluster contains the content clusters storage, not invalidclustername. Please select a valid vespa cluster."); + + ClientParameters params = createParameters() + .setCluster("invalidclustername") + .build(); + + ClusterList clusterList = new ClusterList(Collections.singletonList(new ClusterDef("storage", "content/cluster.foo/storage"))); + + DocumentRetriever documentRetriever = createDocumentRetriever(params, clusterList); + documentRetriever.retrieveDocuments(); + } + + @Test + public void testEmtpyClusterList() throws DocumentRetrieverException { + exception.expect(DocumentRetrieverException.class); + exception.expectMessage("The Vespa cluster does not have any content clusters declared."); + + ClientParameters params = createParameters() + .setCluster("invalidclustername") + .build(); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + } + + @Test + public void testHandlingErrorFromMessageBus() throws DocumentRetrieverException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1)) + .build(); + + Reply r = new GetDocumentReply(null); + r.addError(new Error(0, "Error message")); + when(mockedSession.syncSend(any())).thenReturn(r); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + + assertTrue(errContent.toString().contains("Request failed")); + } + + @Test + public void testShowDocSize() throws DocumentRetrieverException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1)) + .setShowDocSize(true) + .build(); + + Document document = new Document(DataType.DOCUMENT, new DocumentId(DOC_ID_1)); + when(mockedSession.syncSend(any())).thenReturn(new GetDocumentReply(document)); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + + assertTrue(outContent.toString().contains(String.format("Document size: %d bytes", document.getSerializedSize()))); + } + + @Test + public void testPrintIdOnly() throws DocumentRetrieverException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1)) + .setPrintIdsOnly(true) + .build(); + + when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1)); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + + assertEquals(DOC_ID_1 + "\n", outContent.toString()); + } + + @Test + public void testDocumentNotFound() throws DocumentRetrieverException { + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1)) + .setPrintIdsOnly(true) + .build(); + + when(mockedSession.syncSend(any())).thenReturn(new GetDocumentReply(null)); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + + verify(mockedSession, times(1)).syncSend(any()); + assertEquals(outContent.toString(), "Document not found.\n"); + } + + @Test + public void testTrace() throws DocumentRetrieverException { + final int traceLevel = 9; + ClientParameters params = createParameters() + .setDocumentIds(asIterator(DOC_ID_1)) + .setTraceLevel(traceLevel) + .build(); + + GetDocumentReply reply = new GetDocumentReply(new Document(DataType.DOCUMENT, new DocumentId(DOC_ID_1))); + reply.getTrace().getRoot().addChild("childnode"); + when(mockedSession.syncSend(any())).thenReturn(reply); + + DocumentRetriever documentRetriever = createDocumentRetriever(params); + documentRetriever.retrieveDocuments(); + + verify(mockedSession, times(1)).setTraceLevel(traceLevel); + assertTrue(outContent.toString().contains("<trace>")); + } + +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java new file mode 100644 index 00000000000..2df8b2c4751 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsPrinterTest.java @@ -0,0 +1,87 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +import com.yahoo.document.BucketId; +import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class BucketStatsPrinterTest { + + private BucketStatsRetriever retriever; + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + + @Before + public void mockBucketStatsRetriever() throws BucketStatsException { + retriever = mock(BucketStatsRetriever.class); + when(retriever.getBucketIdForType(any(), any())).thenReturn(new BucketId(0x42)); + when(retriever.retrieveBucketList(any())).thenReturn(Collections.emptyList()); + when(retriever.retrieveBucketStats(any(), any(), any())).thenReturn(""); + } + + @After + public void resetOutputMock() { + out.reset(); + } + + private String getOutputString() { + String content = out.toString(); + out.reset(); + return content; + } + + private String retreiveAndPrintBucketStats(ClientParameters.SelectionType type, String id, boolean dumpData) throws BucketStatsException { + BucketStatsPrinter printer = new BucketStatsPrinter(retriever, new PrintStream(out)); + printer.retrieveAndPrintBucketStats(type, id, dumpData); + return getOutputString(); + } + + @Test + public void testShouldPrintBucketIdForUserAndGroup() throws BucketStatsException { + String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", false); + assertTrue(output.contains("Generated 32-bit bucket id")); + + output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.GROUP, "mygroup", false); + assertTrue(output.contains("Generated 32-bit bucket id")); + } + + @Test + public void testShouldPrintWarningIfBucketListEmpty() throws BucketStatsException { + String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", false); + assertTrue(output.contains("No actual files were stored for this bucket")); + } + + @Test + public void testShouldPrintBucketList() throws BucketStatsException { + List<GetBucketListReply.BucketInfo> bucketList = new ArrayList<>(); + String dummyInfoString = "dummyinformation"; + bucketList.add(new GetBucketListReply.BucketInfo(new BucketId(0), dummyInfoString)); + when(retriever.retrieveBucketList(any())).thenReturn(bucketList); + + String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", false); + assertTrue(output.contains(dummyInfoString)); + } + + @Test + public void testShouldPrintBucketStats() throws BucketStatsException { + String dummyBucketStats = "dummystats"; + GetBucketListReply.BucketInfo bucketInfo = new GetBucketListReply.BucketInfo(new BucketId(0), "dummy"); + when(retriever.retrieveBucketList(any())).thenReturn(Collections.singletonList(bucketInfo)); + when(retriever.retrieveBucketStats(any(), any(), any())).thenReturn(dummyBucketStats); + + String output = retreiveAndPrintBucketStats(ClientParameters.SelectionType.USER, "1234", true); + assertTrue(output.contains(dummyBucketStats)); + } +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java new file mode 100644 index 00000000000..38a79aa7b5d --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/BucketStatsRetrieverTest.java @@ -0,0 +1,141 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +import com.yahoo.document.BucketId; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.DocumentId; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusSyncSession; +import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply; +import com.yahoo.documentapi.messagebus.protocol.StatBucketReply; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.routing.Route; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentMatcher; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class BucketStatsRetrieverTest { + private final BucketIdFactory bucketIdFactory = new BucketIdFactory(); + + private DocumentAccessFactory mockedFactory; + private MessageBusDocumentAccess mockedDocumentAccess; + private MessageBusSyncSession mockedSession; + + + @Before + public void prepareMessageBusMocks() { + this.mockedFactory = mock(DocumentAccessFactory.class); + this.mockedDocumentAccess = mock(MessageBusDocumentAccess.class); + this.mockedSession = mock(MessageBusSyncSession.class); + when(mockedFactory.createDocumentAccess()).thenReturn(mockedDocumentAccess); + when(mockedDocumentAccess.createSyncSession(any())).thenReturn(mockedSession); + } + + @Test + public void testGetBucketId() throws BucketStatsException { + BucketStatsRetriever retriever = createRetriever(); + + assertEquals("BucketId(0x80000000000004d2)", + retriever.getBucketIdForType(ClientParameters.SelectionType.USER, "1234").toString()); + assertEquals("BucketId(0x800000003a7455d7)", + retriever.getBucketIdForType(ClientParameters.SelectionType.GROUP, "mygroup").toString()); + assertEquals("BucketId(0x800000003a7455d7)", + retriever.getBucketIdForType(ClientParameters.SelectionType.BUCKET, "0x800000003a7455d7").toString()); + assertEquals("BucketId(0xeb018ac5e5732db3)", + retriever.getBucketIdForType(ClientParameters.SelectionType.DOCUMENT, "id:ns:type::another").toString()); + assertEquals("BucketId(0xeadd5fe811a2012c)", + retriever.getBucketIdForType(ClientParameters.SelectionType.GID, "0x2c01a21163cb7d0ce85fddd6").toString()); + } + + @Test + public void testRetrieveBucketList() throws BucketStatsException { + String bucketInfo = "I like turtles!"; + BucketId bucketId = bucketIdFactory.getBucketId(new DocumentId("id:ns:type::another")); + + GetBucketListReply reply = new GetBucketListReply(); + reply.getBuckets().add(new GetBucketListReply.BucketInfo(bucketId, bucketInfo)); + when(mockedSession.syncSend(any())).thenReturn(reply); + + List<GetBucketListReply.BucketInfo> bucketList = createRetriever().retrieveBucketList(bucketId); + + verify(mockedSession, times(1)).syncSend(any()); + assertEquals(1, bucketList.size()); + assertEquals(bucketInfo, bucketList.get(0).getBucketInformation()); + } + + @Test + public void testRetrieveBucketStats() throws BucketStatsException { + String docId = "id:ns:type::another"; + String bucketInfo = "I like turtles!"; + BucketId bucketId = bucketIdFactory.getBucketId(new DocumentId(docId)); + + StatBucketReply reply = new StatBucketReply(); + reply.setResults(bucketInfo); + when(mockedSession.syncSend(any())).thenReturn(reply); + String result = createRetriever().retrieveBucketStats(ClientParameters.SelectionType.DOCUMENT, docId, bucketId); + + verify(mockedSession, times(1)).syncSend(any()); + assertEquals(bucketInfo, result); + } + + @Test + public void testShutdownHook() { + class MockShutdownRegistrar implements BucketStatsRetriever.ShutdownHookRegistrar { + public Runnable shutdownRunnable; + @Override + public void registerShutdownHook(Runnable runnable) { + shutdownRunnable = runnable; + } + } + MockShutdownRegistrar registrar = new MockShutdownRegistrar(); + new BucketStatsRetriever(mockedFactory, "default", registrar); + registrar.shutdownRunnable.run(); + + verify(mockedSession, times(1)).destroy(); + verify(mockedDocumentAccess, times(1)).shutdown(); + } + + @Test(expected = BucketStatsException.class) + public void testShouldFailOnReplyError() throws BucketStatsException { + GetBucketListReply reply = new GetBucketListReply(); + reply.addError(new Error(0, "errormsg")); + when(mockedSession.syncSend(any())).thenReturn(reply); + + createRetriever().retrieveBucketList(new BucketId(1)); + } + + @Test + public void testRoute() throws BucketStatsException { + String route = "default"; + BucketId bucketId = bucketIdFactory.getBucketId(new DocumentId("id:ns:type::another")); + GetBucketListReply reply = new GetBucketListReply(); + reply.getBuckets().add(new GetBucketListReply.BucketInfo(bucketId, "I like turtles!")); + when(mockedSession.syncSend(any())).thenReturn(reply); + + BucketStatsRetriever retriever = new BucketStatsRetriever(mockedFactory, route, t -> {}); + retriever.retrieveBucketList(new BucketId(0)); + + verify(mockedSession).syncSend(argThat(new ArgumentMatcher<Message>() { + @Override + public boolean matches(Object o) { + return ((Message) o).getRoute().equals(Route.parse(route)); + } + })); + } + + private BucketStatsRetriever createRetriever() { + return new BucketStatsRetriever(mockedFactory, "default", t -> {}); + } + +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespastat/CommandLineOptionsTest.java b/vespaclient-java/src/test/java/com/yahoo/vespastat/CommandLineOptionsTest.java new file mode 100644 index 00000000000..e90c47e3150 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespastat/CommandLineOptionsTest.java @@ -0,0 +1,78 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.junit.Assert.*; + +public class CommandLineOptionsTest { + + private ClientParameters getParsedOptions(String... args) { + CommandLineOptions parser = new CommandLineOptions(); + return parser.parseCommandLineArguments(args); + } + + @Test + public void testHelp() { + assertTrue(getParsedOptions("--help").help); + } + + @Test + public void testMultipleOptions() { + ClientParameters params = getParsedOptions("--dump", "--route", "dummyroute", "--user", "userid"); + assertTrue(params.dumpData); + assertEquals("dummyroute", params.route); + assertEquals(ClientParameters.SelectionType.USER, params.selectionType); + assertEquals("userid", params.id); + } + + @Test + public void testSelectionTypes() { + assertEquals(ClientParameters.SelectionType.USER, getParsedOptions("--user", "id").selectionType); + assertEquals(ClientParameters.SelectionType.DOCUMENT, getParsedOptions("--document", "id").selectionType); + assertEquals(ClientParameters.SelectionType.BUCKET, getParsedOptions("--bucket", "id").selectionType); + assertEquals(ClientParameters.SelectionType.GROUP, getParsedOptions("--group", "id").selectionType); + assertEquals(ClientParameters.SelectionType.GID, getParsedOptions("--gid", "id").selectionType); + } + + @Test(expected = IllegalArgumentException.class) + public void testMissingSelectionType() { + getParsedOptions(); + } + + @Test(expected = IllegalArgumentException.class) + public void testFailOnMultipleDumpTypes() { + getParsedOptions("--user", "id", "--document", "id", "--group", "id", "--gid", "id"); + } + + @Test + public void testForceDumpOnDocumentOrGid() { + assertTrue(getParsedOptions("--document", "docid").dumpData); + assertTrue(getParsedOptions("--gid", "gid").dumpData); + } + + @Test + public void testPrintHelp() { + ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + PrintStream oldOut = System.out; + System.setOut(new PrintStream(outContent)); + try { + CommandLineOptions options = new CommandLineOptions(); + options.printHelp(); + String output = outContent.toString(); + assertTrue(output.contains("vdsstat [options]")); + } finally { + System.setOut(oldOut); + outContent.reset(); + } + } + + @Test + public void testDefaultRoute() { + assertEquals("default", getParsedOptions("--user", "dummyuser").route); + } + +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTargetTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTargetTestCase.java new file mode 100644 index 00000000000..4eec05f7bc7 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTargetTestCase.java @@ -0,0 +1,56 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespavisit; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class VdsVisitTargetTestCase { + + @Test + public void testParametersSlobrok() throws Exception { + VdsVisitTarget target = new VdsVisitTarget(); + target.parseArguments(new String[]{ + "--bindtoslobrok", "myname", + "--processtime", "34", + "--visithandler", "Foo", + "--visitoptions", "foo bar zoo", + "-i", + "-v" + }); + + assertEquals("myname", target.getSlobrokAddress()); + assertEquals(34, target.getProcessTime()); + assertEquals("Foo", target.getHandlerClassName()); + assertEquals(3, target.getHandlerArgs().length); + assertEquals("foo", target.getHandlerArgs()[0]); + assertEquals("bar", target.getHandlerArgs()[1]); + assertEquals("zoo", target.getHandlerArgs()[2]); + assertTrue(target.isVerbose()); + assertTrue(target.isPrintIds()); + } + + @Test + public void testParametersPort() throws Exception { + VdsVisitTarget target = new VdsVisitTarget(); + target.parseArguments("--bindtosocket 1234".split(" ")); + assertEquals(1234, target.getPort()); + assertEquals(null, target.getSlobrokAddress()); + } + + public void assertException(String params) { + try { + VdsVisitTarget target = new VdsVisitTarget(); + target.parseArguments(params.split(" ")); + assertTrue(false); + } catch (Exception e) { + + } + } + + @Test + public void testPortAndSlobrok() { + assertException("--bindtoslobrok foo --bindtosocket 1234"); + assertException("--bindtoport foo"); + } + +} diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java new file mode 100644 index 00000000000..49060a5715f --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java @@ -0,0 +1,475 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespavisit; + +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.*; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.messagebus.StaticThrottlePolicy; +import com.yahoo.messagebus.Trace; +import com.yahoo.vespaclient.ClusterDef; +import com.yahoo.vespaclient.ClusterList; +import org.apache.commons.cli.Options; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; + +public class VdsVisitTestCase { + + private VdsVisit.ArgumentParser createMockArgumentParser() { + Options opts = VdsVisit.createOptions(); + return new VdsVisit.ArgumentParser(opts); + } + + @Test + public void testCommandLineShortOptions() throws Exception { + // short options testing (for options that do not collide with each other) + String[] args = new String[] { + "-d", "foo.remote", + "-s", "'id.user=1234'", + "-f", "5678", + "-t", "9012", + "-l", "foodoc.bar,foodoc.baz", + "-m", "6000", + "-b", "5", + "-p", "foo-progress.txt", + "-u", "123456789", + "-c", "kittens", + "-r", + "-v" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + VdsVisit.VdsVisitParameters allParams = parser.parse(args); + assertNotNull(allParams); + + VisitorParameters params = allParams.getVisitorParameters(); + assertNotNull(params); + assertEquals("foo.remote", params.getRemoteDataHandler()); + assertEquals("'id.user=1234'", params.getDocumentSelection()); + assertEquals(5678, params.getFromTimestamp()); + assertEquals(9012, params.getToTimestamp()); + assertEquals("foodoc.bar,foodoc.baz", params.getFieldSet()); + assertEquals(6000, params.getMaxPending()); + assertEquals(5, params.getMaxBucketsPerVisitor()); + assertEquals("foo-progress.txt", params.getResumeFileName()); + assertEquals(123456789, params.getTimeoutMs()); + assertEquals(7 * 24 * 60 * 60 * 1000, allParams.getFullTimeout()); + assertEquals("kittens", allParams.getCluster()); + assertTrue(allParams.isVerbose()); + } + + /** + * Test the parameters that could not be used in conjunction with + * those in the first parameter test. + * @throws Exception + */ + @Test + public void testCommandLineShortOptions2() throws Exception { + // Short options testing (for options that do not collide with each other) + String[] args = new String[] { + "-o", "654321", + "-e" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + VdsVisit.VdsVisitParameters allParams = parser.parse(args); + assertNotNull(allParams); + + VisitorParameters params = allParams.getVisitorParameters(); + assertNotNull(params); + assertEquals(654321, allParams.getFullTimeout()); + assertEquals(654321, params.getTimeoutMs()); + assertEquals("[header]", params.getFieldSet()); + } + + @Test + public void testCommandLineShortOptionsPrintIdsOnly() throws Exception { + // Short options testing (for options that do not collide with each other) + String[] args = new String[] { + "-i" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + VdsVisit.VdsVisitParameters allParams = parser.parse(args); + assertNotNull(allParams); + + VisitorParameters params = allParams.getVisitorParameters(); + assertNotNull(params); + assertEquals("[id]", params.getFieldSet()); + assertTrue(allParams.isPrintIdsOnly()); + } + + @Test + public void testCommandLineLongOptions() throws Exception { + // short options testing (for options that do not collide with each other) + String[] args = new String[] { + "--datahandler", "foo.remote", + "--selection", "'id.user=1234'", + "--from", "5678", + "--to", "9012", + "--fieldset", "foodoc.bar,foodoc.baz", + "--maxpending", "6000", + "--maxbuckets", "5", + "--progress", "foo-progress.txt", + "--maxpendingsuperbuckets", "3", + "--buckettimeout", "123456789", + "--cluster", "kittens", + "--visitinconsistentbuckets", + "--visitlibrary", "fnord", + "--libraryparam", "asdf", "rargh", + "--libraryparam", "pinkie", "pie", + "--processtime", "555", + "--maxhits", "1001", + "--maxtotalhits", "2002", + "--tracelevel", "8", + "--priority", "NORMAL_1", + "--ordering", "ascending", + "--skipbucketsonfatalerrors", + "--abortonclusterdown", + "--visitremoves" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + VdsVisit.VdsVisitParameters allParams = parser.parse(args); + assertNotNull(allParams); + + VisitorParameters params = allParams.getVisitorParameters(); + assertNotNull(params); + + assertEquals("foo.remote", params.getRemoteDataHandler()); + assertEquals("'id.user=1234'", params.getDocumentSelection()); + assertEquals(5678, params.getFromTimestamp()); + assertEquals(9012, params.getToTimestamp()); + assertEquals("foodoc.bar,foodoc.baz", params.getFieldSet()); + assertEquals(6000, params.getMaxPending()); + assertEquals(5, params.getMaxBucketsPerVisitor()); + assertEquals("foo-progress.txt", params.getResumeFileName()); + assertEquals(123456789, params.getTimeoutMs()); + assertEquals(7 * 24 * 60 * 60 * 1000, allParams.getFullTimeout()); + assertEquals("kittens", allParams.getCluster()); + + assertTrue(params.getThrottlePolicy() instanceof StaticThrottlePolicy); + assertEquals(3, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount()); + + assertTrue(params.visitInconsistentBuckets()); + assertEquals("fnord", params.getVisitorLibrary()); + // TODO: FIXME? multiple library params doesn't work + assertTrue(Arrays.equals("rargh".getBytes(), params.getLibraryParameters().get("asdf"))); + //assertTrue(Arrays.equals("pie".getBytes(), params.getLibraryParameters().get("pinkie"))); + assertEquals(555, allParams.getProcessTime()); + assertEquals(1001, params.getMaxFirstPassHits()); + assertEquals(2002, params.getMaxTotalHits()); + assertEquals(8, params.getTraceLevel()); + assertEquals(DocumentProtocol.Priority.NORMAL_1, params.getPriority()); + assertEquals(OrderingSpecification.ASCENDING, params.getVisitorOrdering()); + assertTrue(allParams.getAbortOnClusterDown()); + assertTrue(params.visitRemoves()); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + PrintStream printStream = new PrintStream(outputStream); + VdsVisit.verbosePrintParameters(allParams, printStream); + printStream.flush(); + String nl = System.getProperty("line.separator"); // the joys of running tests on windows + assertEquals( + "Time out visitor after 123456789 ms." + nl + + "Visiting documents matching: 'id.user=1234'" + nl + + "Visiting in the inclusive timestamp range 5678 - 9012." + nl + + "Visiting field set foodoc.bar,foodoc.baz." + nl + + "Visiting inconsistent buckets." + nl + + "Including remove entries." + nl + + "Tracking progress in file: foo-progress.txt" + nl + + "Let visitor have maximum 6000 replies pending on data handlers per storage node visitor." + nl + + "Visit maximum 5 buckets per visitor." + nl + + "Sending data to data handler at: foo.remote" + nl + + "Using visitor library 'fnord'." + nl + + "Adding the following library specific parameters:" + nl + + " asdf = rargh" + nl + + "Visitor priority NORMAL_1" + nl + + "Skip visiting super buckets with fatal errors." + nl, + outputStream.toString("utf-8")); + + args = new String[] { + "--ordering", "descending" + }; + allParams = parser.parse(args); + params = allParams.getVisitorParameters(); + assertEquals(OrderingSpecification.DESCENDING, params.getVisitorOrdering()); + } + + private static String[] emptyArgList() { return new String[]{}; } + + @Test + public void visitor_priority_is_low1_by_default() throws Exception { + VdsVisit.VdsVisitParameters allParams = createMockArgumentParser().parse(emptyArgList()); + + VisitorParameters params = allParams.getVisitorParameters(); + assertEquals(DocumentProtocol.Priority.LOW_1, params.getPriority()); + } + + @Test + public void testBadPriorityValue() throws Exception { + String[] args = new String[] { + "--priority", "super_hyper_important" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + try { + parser.parse(args); + fail("no exception thrown"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Unknown priority name")); + } + } + + @Test + public void testBadOrderingValue() throws Exception { + String[] args = new String[] { + "--ordering", "yonder" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + try { + parser.parse(args); + fail("no exception thrown"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Unknown ordering")); + } + } + + @Test + public void testCommandLineShortOptionsInvokeHelp() throws Exception { + // Short options testing (for options that do not collide with each other) + String[] args = new String[] { + "-h" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + VdsVisit.VdsVisitParameters allParams = parser.parse(args); + assertNull(allParams); + } + + @Test + public void testAutoSelectClusterRoute() throws Exception { + List<ClusterDef> clusterDefs = new ArrayList<>(); + clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage")); + ClusterList clusterList = new ClusterList(clusterDefs); + + String route = VdsVisit.resolveClusterRoute(clusterList, null); + assertEquals("[Storage:cluster=storage;clusterconfigid=content/cluster.foo/storage]", route); + } + + @Test + public void testBadClusterName() throws Exception { + List<ClusterDef> clusterDefs = new ArrayList<>(); + clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage")); + ClusterList clusterList = new ClusterList(clusterDefs); + try { + VdsVisit.resolveClusterRoute(clusterList, "borkbork"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Your vespa cluster contains the content clusters storage, not borkbork.")); + } + } + + @Test + public void testRequireClusterOptionIfMultipleClusters() { + List<ClusterDef> clusterDefs = new ArrayList<>(); + clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage")); + clusterDefs.add(new ClusterDef("storage2", "content/cluster.bar/storage")); + ClusterList clusterList = new ClusterList(clusterDefs); + try { + VdsVisit.resolveClusterRoute(clusterList, null); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Please use the -c option to select one of them")); + } + } + + @Test + public void testExplicitClusterOptionWithMultipleClusters() { + List<ClusterDef> clusterDefs = new ArrayList<>(); + clusterDefs.add(new ClusterDef("storage", "content/cluster.foo/storage")); + clusterDefs.add(new ClusterDef("storage2", "content/cluster.bar/storage")); + ClusterList clusterList = new ClusterList(clusterDefs); + + String route = VdsVisit.resolveClusterRoute(clusterList, "storage2"); + assertEquals("[Storage:cluster=storage2;clusterconfigid=content/cluster.bar/storage]", route); + } + + @Test + public void testFailIfNoContentClustersAvailable() { + List<ClusterDef> clusterDefs = new ArrayList<>(); + ClusterList clusterList = new ClusterList(clusterDefs); + try { + VdsVisit.resolveClusterRoute(clusterList, null); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Your Vespa cluster does not have any content clusters")); + } + } + + @Test + public void testStatistics() throws Exception { + String[] args = new String[] { + "--statistics", "foo" + }; + VdsVisit.ArgumentParser parser = createMockArgumentParser(); + VdsVisit.VdsVisitParameters allParams = parser.parse(args); + assertNotNull(allParams); + + VisitorParameters params = allParams.getVisitorParameters(); + assertNotNull(params); + assertEquals("foo", allParams.getStatisticsParts()); + assertEquals("[id]", params.getFieldSet()); + assertEquals("CountVisitor", params.getVisitorLibrary()); + } + + // TODO: use DummyVisitorSession instead? + private static class MockVisitorSession implements VisitorSession { + private VisitorParameters params; + + public MockVisitorSession(VisitorParameters params) { + this.params = params; + params.getLocalDataHandler().setSession(this); + } + + @Override + public boolean isDone() { + return true; + } + + @Override + public ProgressToken getProgress() { + return null; + } + + @Override + public Trace getTrace() { + return null; + } + + @Override + public boolean waitUntilDone(long l) throws InterruptedException { + params.getControlHandler().onDone(VisitorControlHandler.CompletionCode.SUCCESS, "woo!"); + // Return immediately + return true; + } + + @Override + public void ack(AckToken ackToken) { + } + + @Override + public void abort() { + } + + @Override + public VisitorResponse getNext() { + return null; + } + + @Override + public VisitorResponse getNext(int i) throws InterruptedException { + return null; + } + + @Override + public void destroy() { + } + } + + private static class MockVisitorSessionAccessor implements VdsVisit.VisitorSessionAccessor { + boolean shutdown = false; + @Override + public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException { + return new MockVisitorSession(params); + } + + @Override + public void shutdown() { + shutdown = true; + } + + public boolean isShutdown() { + return shutdown; + } + } + + private static class MockVisitorSessionAccessorFactory implements VdsVisit.VisitorSessionAccessorFactory { + + private MockVisitorSessionAccessor lastCreatedAccessor = null; + + @Override + public VdsVisit.VisitorSessionAccessor createVisitorSessionAccessor() { + lastCreatedAccessor = new MockVisitorSessionAccessor(); + return lastCreatedAccessor; + } + + public MockVisitorSessionAccessor getLastCreatedAccessor() { + return lastCreatedAccessor; + } + } + + private static class MockShutdownHookRegistrar implements VdsVisit.ShutdownHookRegistrar { + Thread cleanUpThread; + + @Override + public void registerShutdownHook(Thread thread) { + cleanUpThread = thread; + } + + public Thread getCleanUpThread() { + return cleanUpThread; + } + } + + @Test + public void testVdsVisitRunLogic() { + MockVisitorSessionAccessorFactory accessorFactory = new MockVisitorSessionAccessorFactory(); + MockShutdownHookRegistrar shutdownHookRegistrar = new MockShutdownHookRegistrar(); + VdsVisit vdsVisit = new VdsVisit(accessorFactory, shutdownHookRegistrar); + + VdsVisit.VdsVisitParameters params = new VdsVisit.VdsVisitParameters(); + VisitorParameters visitorParameters = new VisitorParameters(""); + params.setVisitorParameters(visitorParameters); + + visitorParameters.setResumeFileName("src/test/files/progress.txt"); + vdsVisit.setVdsVisitParameters(params); + + int code = vdsVisit.doRun(); + assertEquals(0, code); + + assertNotNull(shutdownHookRegistrar.getCleanUpThread()); + shutdownHookRegistrar.getCleanUpThread().run(); + + assertNotNull(accessorFactory.getLastCreatedAccessor()); + assertTrue(accessorFactory.getLastCreatedAccessor().isShutdown()); + + // Ensure progress token stuff was read from file + ProgressToken progress = visitorParameters.getResumeToken(); + assertNotNull(progress); + assertEquals(14, progress.getDistributionBitCount()); + assertEquals(3, progress.getPendingBucketCount()); + } + + @Test + public void testVdsVisitRunLogicProgressFileNotYetCreated() { + MockVisitorSessionAccessorFactory accessorFactory = new MockVisitorSessionAccessorFactory(); + MockShutdownHookRegistrar shutdownHookRegistrar = new MockShutdownHookRegistrar(); + VdsVisit vdsVisit = new VdsVisit(accessorFactory, shutdownHookRegistrar); + + VdsVisit.VdsVisitParameters params = new VdsVisit.VdsVisitParameters(); + VisitorParameters visitorParameters = new VisitorParameters(""); + params.setVisitorParameters(visitorParameters); + + visitorParameters.setResumeFileName("src/test/files/progress-not-existing.txt"); + vdsVisit.setVdsVisitParameters(params); + + // Should not fail with file not found + int code = vdsVisit.doRun(); + assertEquals(0, code); + + assertNotNull(shutdownHookRegistrar.getCleanUpThread()); + shutdownHookRegistrar.getCleanUpThread().run(); + + assertNotNull(accessorFactory.getLastCreatedAccessor()); + assertTrue(accessorFactory.getLastCreatedAccessor().isShutdown()); + } +} |