From 30fb044bf2deb88edbb1056ce670c92427e06d37 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Wed, 17 Jan 2018 12:17:47 +0100 Subject: Add and propagate visitor parameter bucket space Command line option for bucket space added to vespavisit binary. --- .../java/com/yahoo/documentapi/VisitorParameters.java | 10 +++++++++- .../messagebus/MessageBusVisitorSession.java | 1 + .../messagebus/protocol/CreateVisitorMessage.java | 10 ++++++++++ .../yahoo/documentapi/VisitorParametersTestCase.java | 2 ++ .../messagebus/protocol/test/Messages50TestCase.java | 3 +++ .../test/MessageBusVisitorSessionTestCase.java | 19 +++++++++++++++++++ 6 files changed, 44 insertions(+), 1 deletion(-) (limited to 'documentapi') diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java index 01abd6454a9..0efdefa197e 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java @@ -20,6 +20,7 @@ import java.util.TreeMap; public class VisitorParameters extends Parameters { private String documentSelection; + private String bucketSpace = "default"; private String visitorLibrary = "DumpVisitor"; private int maxPending = 32; private long timeoutMs = -1; @@ -69,6 +70,7 @@ public class VisitorParameters extends Parameters { */ public VisitorParameters(VisitorParameters params) { setDocumentSelection(params.getDocumentSelection()); + setBucketSpace(params.getBucketSpace()); setVisitorLibrary(params.getVisitorLibrary()); setMaxPending(params.getMaxPending()); setTimeoutMs(params.getTimeoutMs()); @@ -108,6 +110,9 @@ public class VisitorParameters extends Parameters { /** @return The selection string used for visiting. */ public String getDocumentSelection() { return documentSelection; } + /** @return The bucket space to visit */ + public String getBucketSpace() { return bucketSpace; } + /** @return What visitor library to use for the visiting. The library in question must be installed on each storage node in the target cluster. */ public String getVisitorLibrary() { return visitorLibrary; } @@ -192,6 +197,9 @@ public class VisitorParameters extends Parameters { /** Set the document selection expression */ public void setDocumentSelection(String selection) { documentSelection = selection; } + /** Set which (single) bucket space this visiting will be against. */ + public void setBucketSpace(String bucketSpace) { this.bucketSpace = bucketSpace; } + /** Set which visitor library is used for visiting in storage. DumpVisitor is most common implementation. */ public void setVisitorLibrary(String library) { visitorLibrary = library; } @@ -343,11 +351,11 @@ public class VisitorParameters extends Parameters { this.dynamicMaxBucketsIncreaseFactor = dynamicMaxBucketsIncreaseFactor; } - // Inherit docs from Object public String toString() { StringBuffer sb = new StringBuffer(); sb.append("VisitorParameters(\n") .append(" Document selection: ").append(documentSelection).append('\n') + .append(" Bucket space: ").append(bucketSpace).append('\n') .append(" Visitor library: ").append(visitorLibrary).append('\n') .append(" Max pending: ").append(maxPending).append('\n') .append(" Timeout (ms): ").append(timeoutMs).append('\n') diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java index 4dd78278df0..a146caa972e 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -595,6 +595,7 @@ public class MessageBusVisitorSession implements VisitorSession { msg.setTimeRemaining(messageTimeoutMs); msg.setBuckets(Arrays.asList(bucket.getSuperbucket(), bucket.getProgress())); msg.setDocumentSelection(params.getDocumentSelection()); + msg.setBucketSpace(params.getBucketSpace()); msg.setFromTimestamp(params.getFromTimestamp()); msg.setToTimestamp(params.getToTimestamp()); msg.setMaxPendingReplyCount(params.getMaxPending()); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java index 57384836001..e6d51103b3e 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java @@ -12,6 +12,7 @@ public class CreateVisitorMessage extends DocumentMessage { private String controlDestination = ""; private String dataDestination = ""; private String docSelection = ""; + private String bucketSpace = "default"; private int maxPendingReplyCount = 8; private List buckets = new ArrayList<>(); private long fromTime = 0; @@ -76,6 +77,14 @@ public class CreateVisitorMessage extends DocumentMessage { docSelection = documentSelection; } + public String getBucketSpace() { + return bucketSpace; + } + + public void setBucketSpace(String bucketSpace) { + this.bucketSpace = bucketSpace; + } + public int getMaxPendingReplyCount() { return maxPendingReplyCount; } @@ -202,6 +211,7 @@ public class CreateVisitorMessage extends DocumentMessage { sb.append(", time ").append(fromTime).append('-').append(toTime); } sb.append(", selection '").append(docSelection).append('\''); + sb.append(", bucket space '").append(bucketSpace).append('\''); if (!libName.equals("DumpVisitor")) { sb.append(", library ").append(libName); } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java index 54b12b0963d..181253c4377 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java @@ -12,6 +12,7 @@ public class VisitorParametersTestCase { private VisitorParameters createVisitorParameters() { VisitorParameters params = new VisitorParameters(""); params.setDocumentSelection("id.user==5678"); + params.setBucketSpace("narnia"); params.setFromTimestamp(9001); params.setToTimestamp(10001); params.setVisitorLibrary("CoolVisitor"); @@ -89,6 +90,7 @@ public class VisitorParametersTestCase { assertEquals( "VisitorParameters(\n" + " Document selection: id.user==5678\n" + + " Bucket space: narnia\n" + " Visitor library: CoolVisitor\n" + " Max pending: 111\n" + " Timeout (ms): 1337\n" + diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java index f0b76907fc4..0ef83c6b5af 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java @@ -190,6 +190,7 @@ public class Messages50TestCase extends MessagesTestBase { assertEquals("CreateVisitorMessage(" + "No buckets, " + "selection 'true and false or true', " + + "bucket space 'default', " + "library SomeLibrary, including removes, " + "get fields: [all]" + ")", @@ -200,6 +201,7 @@ public class Messages50TestCase extends MessagesTestBase { assertEquals("CreateVisitorMessage(" + "Bucket BucketId(0x40000000000004d2), " + "selection 'true and false or true', " + + "bucket space 'default', " + "library SomeLibrary, including removes, " + "get fields: [all]" + ")", @@ -216,6 +218,7 @@ public class Messages50TestCase extends MessagesTestBase { "5 buckets: BucketId(0x40000000000004d2) BucketId(0x40000000000004d3) BucketId(0x40000000000004d4) ..., " + "time 10001-20002, " + "selection 'true and false or true', " + + "bucket space 'default', " + "library SomeLibrary, including removes, " + "get fields: [all], " + "visit inconsistent buckets" + diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java index 25116273377..874066d50a3 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusVisitorSessionTestCase.java @@ -2560,6 +2560,25 @@ public class MessageBusVisitorSessionTestCase { mc.controlHandler.toString()); } + @Test + public void visit_default_bucket_space_unless_explicitly_given() { + MockComponents mc = createDefaultMock(""); + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + CreateVisitorMessage cmd = (CreateVisitorMessage)mc.sender.getAndRemoveMessage(0); + assertEquals("default", cmd.getBucketSpace()); + } + + @Test + public void explicitly_provided_bucket_space_is_propagated_to_visitor_commands() { + MockComponents mc = createDefaultMock(""); + mc.params.setBucketSpace("upside down"); + mc.visitorSession.start(); + mc.executor.expectAndProcessTasks(1); + CreateVisitorMessage cmd = (CreateVisitorMessage)mc.sender.getAndRemoveMessage(0); + assertEquals("upside down", cmd.getBucketSpace()); + } + /** * TODOs: * - parameter validation (max pending, ...) -- cgit v1.2.3