From 8f2d9c36cccb34a1594d5f1cf82ac852ad5e712d Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Tue, 6 Jun 2017 14:13:55 +0200 Subject: Move vespaclient-java to Vespa open-source --- .../com/yahoo/dummyreceiver/DummyReceiver.java | 196 +++++ .../main/java/com/yahoo/vespafeeder/Arguments.java | 191 +++++ .../vespafeeder/BenchmarkProgressPrinter.java | 76 ++ .../java/com/yahoo/vespafeeder/FileRequest.java | 14 + .../com/yahoo/vespafeeder/InputStreamRequest.java | 38 + .../com/yahoo/vespafeeder/ProgressPrinter.java | 149 ++++ .../java/com/yahoo/vespafeeder/VespaFeeder.java | 171 +++++ .../java/com/yahoo/vespaget/ClientParameters.java | 160 +++++ .../com/yahoo/vespaget/CommandLineOptions.java | 263 +++++++ .../com/yahoo/vespaget/DocumentAccessFactory.java | 17 + .../java/com/yahoo/vespaget/DocumentRetriever.java | 207 ++++++ .../yahoo/vespaget/DocumentRetrieverException.java | 14 + .../src/main/java/com/yahoo/vespaget/Main.java | 46 ++ .../com/yahoo/vespastat/BucketStatsException.java | 18 + .../com/yahoo/vespastat/BucketStatsPrinter.java | 59 ++ .../com/yahoo/vespastat/BucketStatsRetriever.java | 176 +++++ .../java/com/yahoo/vespastat/ClientParameters.java | 73 ++ .../com/yahoo/vespastat/CommandLineOptions.java | 139 ++++ .../com/yahoo/vespastat/DocumentAccessFactory.java | 15 + .../src/main/java/com/yahoo/vespastat/Main.java | 38 + .../VespaSummaryBenchmark.java | 162 +++++ .../com/yahoo/vespavisit/StdOutVisitorHandler.java | 292 ++++++++ .../main/java/com/yahoo/vespavisit/VdsVisit.java | 789 +++++++++++++++++++++ .../java/com/yahoo/vespavisit/VdsVisitHandler.java | 181 +++++ .../java/com/yahoo/vespavisit/VdsVisitTarget.java | 286 ++++++++ .../src/main/sh/vds-document-statistics.sh | 20 + vespaclient-java/src/main/sh/vdsstat.sh | 13 + .../src/main/sh/vespa-query-profile-dump-tool.sh | 6 + .../src/main/sh/vespa-summary-benchmark.sh | 15 + vespaclient-java/src/main/sh/vespadestination.sh | 12 + vespaclient-java/src/main/sh/vespafeeder.sh | 15 + vespaclient-java/src/main/sh/vespaget.sh | 14 + vespaclient-java/src/main/sh/vespavisit.1 | 159 +++++ vespaclient-java/src/main/sh/vespavisit.sh | 14 + vespaclient-java/src/main/sh/vespavisittarget.1 | 40 ++ vespaclient-java/src/main/sh/vespavisittarget.sh | 13 + 36 files changed, 4091 insertions(+) create mode 100755 vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java create mode 100755 vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java create mode 100755 vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentAccessFactory.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetrieverException.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsException.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsPrinter.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespastat/ClientParameters.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespastat/CommandLineOptions.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespastat/DocumentAccessFactory.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java create mode 100644 vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java create mode 100755 vespaclient-java/src/main/sh/vds-document-statistics.sh create mode 100644 vespaclient-java/src/main/sh/vdsstat.sh create mode 100755 vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh create mode 100755 vespaclient-java/src/main/sh/vespa-summary-benchmark.sh create mode 100755 vespaclient-java/src/main/sh/vespadestination.sh create mode 100755 vespaclient-java/src/main/sh/vespafeeder.sh create mode 100644 vespaclient-java/src/main/sh/vespaget.sh create mode 100644 vespaclient-java/src/main/sh/vespavisit.1 create mode 100755 vespaclient-java/src/main/sh/vespavisit.sh create mode 100644 vespaclient-java/src/main/sh/vespavisittarget.1 create mode 100755 vespaclient-java/src/main/sh/vespavisittarget.sh (limited to 'vespaclient-java/src/main') diff --git a/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java new file mode 100755 index 00000000000..7b104455f15 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java @@ -0,0 +1,196 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.dummyreceiver; + +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.documentapi.ThroughputLimitQueue; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.log.LogSetup; +import com.yahoo.messagebus.DestinationSession; +import com.yahoo.messagebus.EmptyReply; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageHandler; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.network.Identity; +import com.yahoo.messagebus.network.rpc.RPCNetworkParams; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import static java.lang.System.out; + +public class DummyReceiver implements MessageHandler { + String name = null; + DestinationSession session; + MessageBusDocumentAccess da; + long sleepTime = 0; + long messageCount = 0; + int maxPendingCount = 0; + long silentNum = 0; + boolean instant = false; + ThreadPoolExecutor executor = null; + int threads = 10; + long maxQueueTime = -1; + BlockingQueue queue; + boolean verbose = false; + + DummyReceiver() { + } + + public class Task implements Runnable { + Reply reply; + + public Task(Reply reply) { + this.reply = reply; + } + + public void run() { + if (sleepTime > 0) { + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + session.reply(reply); + } + } + + public void init() { + MessageBusParams params = new MessageBusParams(new LoadTypeSet()); + params.setRPCNetworkParams(new RPCNetworkParams().setIdentity(new Identity(name))); + params.setDocumentManagerConfigId("client"); + params.getMessageBusParams().setMaxPendingCount(maxPendingCount); + params.getMessageBusParams().setMaxPendingSize(0); + da = new MessageBusDocumentAccess(params); + queue = (maxQueueTime < 0) ? new LinkedBlockingDeque<>() : new ThroughputLimitQueue<>(maxQueueTime); + session = da.getMessageBus().createDestinationSession("default", true, this); + executor = new ThreadPoolExecutor(threads, threads, 5, TimeUnit.SECONDS, queue, new DaemonThreadFactory()); + System.out.println("Registered listener at " + name + "/default with " + maxPendingCount + " max pending and sleep time of " + sleepTime); + } + + public void handleMessage(Message message) { + messageCount++; + if ( silentNum == 0 ) { + System.out.println("Received message " + message + ". Received " + messageCount + " messages so far. In queue size " + queue.size()); + + if (verbose) { + if (message instanceof PutDocumentMessage) { + System.out.println(" Document:\n" + ((PutDocumentMessage) message).getDocumentPut().getDocument().toXML(" ")); + } else if (message instanceof RemoveDocumentMessage) { + System.out.println(" Document id: " + ((RemoveDocumentMessage) message).getDocumentId()); + } else if (message instanceof UpdateDocumentMessage) { + System.out.println(" Update:\n " + ((UpdateDocumentMessage) message).getDocumentUpdate().toString()); + } + } + } else { + if ((messageCount % silentNum) == 0) { + System.out.println("Received " + messageCount + " messages so far. In queue size " + queue.size()); + } + } + + EmptyReply reply = new EmptyReply(); + message.swapState(reply); + + if ( ! instant ) { + try { + executor.execute(new Task(reply)); + } catch (RejectedExecutionException e) { + reply.addError(new Error(ErrorCode.SESSION_BUSY, "Session " + name + "/default is busy")); + session.reply(reply); + } + } else { + session.reply(reply); + } + } + + String getParam(List args, String arg) throws IllegalArgumentException { + try { + return args.remove(0); + } catch (Exception e) { + System.err.println("--" + arg + " requires an argument"); + throw new IllegalArgumentException(arg); + } + } + + public void help() { + out.println("Simple receiver for messagebus messages. Prints the messages received to stdout.\n" + + "\n" + + "The options are:\n" + + " --instant Reply in message thread." + + " --name arg Slobrok name to register\n" + + " --maxqueuetime arg Adjust the in queue size to have a maximum queue wait period of this many ms (default -1 = unlimited)\n" + + " --silent #nummsg Do not dump anything, but progress every #nummsg\n" + + " --sleeptime arg The number of milliseconds to sleep per message, to simulate processing time\n" + + " --threads arg The number of threads to process the incoming data\n" + + " --verbose If set, dump the contents of certain messages to stdout"); + } + + boolean parseArgs(List args) { + try { + while (!args.isEmpty()) { + String arg = args.remove(0); + + if (arg.equals("-h") || arg.equals("--help")) { + help(); + return false; + } else if ("--name".equals(arg)) { + name = getParam(args, arg); + } else if ("--sleeptime".equals(arg)) { + sleepTime = Long.parseLong(getParam(args, arg)); + } else if ("--instant".equals(arg)) { + instant = true; + } else if ("--silent".equals(arg)) { + silentNum = Long.parseLong(getParam(args, arg)); + } else if ("--maxqueuetime".equals(arg)) { + maxQueueTime = Long.parseLong(getParam(args, arg)); + } else if ("--threads".equals(arg)) { + threads = Integer.parseInt(getParam(args, arg)); + } else if ("--verbose".equals(arg)) { + verbose = true; + } else { + help(); + return false; + } + } + + return true; + } catch (IllegalArgumentException e) { + return false; + } + } + + + public static void main(String[] args) { + LogSetup.initVespaLogging("dummyreceiver"); + DummyReceiver rcv = new DummyReceiver(); + + List l = new LinkedList<>(); + for (String arg : args) { + l.add(arg); + } + if (!rcv.parseArgs(l)) { + System.exit(1); + } + + rcv.init(); + while (true) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java new file mode 100644 index 00000000000..249ed1fd70c --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java @@ -0,0 +1,191 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.feedapi.DummySessionFactory; +import com.yahoo.feedapi.MessageBusSessionFactory; +import com.yahoo.feedapi.MessagePropertyProcessor; +import com.yahoo.feedapi.SessionFactory; +import com.yahoo.vespaclient.config.FeederConfig; + +import java.io.BufferedOutputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import static java.lang.System.out; + +/** + * Argument parsing class for the vespa feeder. + */ +public class Arguments { + public FeederConfig getFeederConfig() { + return new FeederConfig(feederConfigBuilder); + } + + public List getFiles() { + return files; + } + + public String getMode() { + return mode; + } + + public boolean isVerbose() { + return verbose; + } + + private FeederConfig.Builder feederConfigBuilder = new FeederConfig.Builder(); + private List files = new ArrayList(); + private String dumpDocumentsFile = null; + private String mode = "standard"; + private boolean validateOnly = false; + private boolean verbose = false; + SessionFactory sessionFactory = null; + MessagePropertyProcessor propertyProcessor = null; + private String priority = null; + + public MessagePropertyProcessor getPropertyProcessor() { + return propertyProcessor; + } + + public void help() { + out.println("This is a tool for feeding xml (deprecated) or json data to a Vespa application.\n" + + "\n" + + "The options are:\n" + + " --abortondataerror arg (true) Whether or not to abort if the xml input has \n" + + " errors (true|false).\n" + + " --abortonsenderror arg (true) Whether or not to abort if an error occured while\n" + + " sending operations to Vespa (true|false).\n" + + " --file arg The name of the input files to read. These can \n" + + " also be passed as arguments without the option \n" + + " prefix. If none is given, this tool parses \n" + + " identifiers from standard in.\n" + + " -h [ --help ] Shows this help page.\n" + + " --maxpending arg The maximum number of operations that are allowed\n" + + " to be pending at any given time. NOTE: This disables dynamic throttling. Use with care.\n" + + " --maxpendingsize arg The maximum size (in bytes) of operations that \n" + + " are allowed to be pending at any given time. \n" + + " --maxfeedrate arg Limits the feed rate to the given number (operations/second). You may still want to increase\n" + + " the max pending size if your feed rate doesn't reach the desired number.\n" + + " --mode arg (=standard) The mode to run vespafeeder in (standard | benchmark).\n" + + " --noretry Turns off retries of recoverable failures.\n" + + " --retrydelay arg (=1) The time (in seconds) to wait between retries of \n" + + " a failed operation.\n" + + " --route arg (=default) The route to send the data to.\n" + + " --timeout arg (=180) The time (in seconds) allowed for sending \n" + + " operations.\n" + + " --trace arg (=0) The trace level of network traffic.\n" + + " --validate Run validation tool on input files instead of \n" + + " feeding them.\n" + + " --dumpDocuments Specify a file where documents in the put are serialized.\n" + + " --priority arg Specify priority of sent messages (see documentation for priority values)\n" + + " --create-if-non-existent Enable setting of create-if-non-existent to true on all document updates in the given xml feed.\n" + + " -v [ --verbose ] Enable verbose output of progress.\n"); + } + + public class HelpShownException extends Exception { + + } + + public Arguments(String[] argList, SessionFactory factory) throws HelpShownException, FileNotFoundException { + parse(argList); + + if (factory != null) { + sessionFactory = factory; + } else if (validateOnly) { + if (dumpDocumentsFile != null) { + BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(dumpDocumentsFile)); + sessionFactory = new DummySessionFactory(null, out); + } else { + sessionFactory = new DummySessionFactory(null, null); + } + } else { + sessionFactory = new MessageBusSessionFactory(propertyProcessor); + } + } + + void parse(String[] argList) throws HelpShownException { + List args = new LinkedList(); + args.addAll(Arrays.asList(argList)); + + while (!args.isEmpty()) { + String arg = args.remove(0); + + if (arg.equals("-h") || arg.equals("--help")) { + help(); + throw new HelpShownException(); + } else if ("--abortondataerror".equals(arg)) { + feederConfigBuilder.abortondocumenterror(getBoolean(getParam(args, arg))); + } else if ("--abortonsenderror".equals(arg)) { + feederConfigBuilder.abortonsenderror(getBoolean(getParam(args, arg))); + } else if ("--file".equals(arg)) { + files.add(getParam(args, arg)); + } else if ("--maxpending".equals(arg)) { + feederConfigBuilder.maxpendingdocs(Integer.parseInt(getParam(args, arg))); + } else if ("--maxpendingsize".equals(arg)) { + feederConfigBuilder.maxpendingbytes(Integer.parseInt(getParam(args, arg))); + } else if ("--mode".equals(arg)) { + mode = getParam(args, arg); + } else if ("--noretry".equals(arg)) { + feederConfigBuilder.retryenabled(false); + } else if ("--retrydelay".equals(arg)) { + feederConfigBuilder.retrydelay(Integer.parseInt(getParam(args, arg))); + } else if ("--route".equals(arg)) { + feederConfigBuilder.route(getParam(args, arg)); + } else if ("--timeout".equals(arg)) { + feederConfigBuilder.timeout(Double.parseDouble(getParam(args, arg))); + } else if ("--trace".equals(arg)) { + feederConfigBuilder.tracelevel(Integer.parseInt(getParam(args, arg))); + } else if ("--validate".equals(arg)) { + validateOnly = true; + } else if ("--dumpDocuments".equals(arg)) { + dumpDocumentsFile = getParam(args, arg); + } else if ("--maxfeedrate".equals(arg)) { + feederConfigBuilder.maxfeedrate(Double.parseDouble(getParam(args, arg))); + } else if ("--create-if-non-existent".equals(arg)) { + feederConfigBuilder.createifnonexistent(true); + } else if ("-v".equals(arg) || "--verbose".equals(arg)) { + verbose = true; + } else if ("--priority".equals(arg)) { + priority = getParam(args, arg); + } else { + files.add(arg); + } + } + + propertyProcessor = new MessagePropertyProcessor(getFeederConfig(), new LoadTypeConfig(new LoadTypeConfig.Builder())); + } + + private String getParam(List args, String arg) throws IllegalArgumentException { + try { + return args.remove(0); + } catch (Exception e) { + System.err.println("--" + arg + " requires an argument"); + throw new IllegalArgumentException(arg); + } + } + + private Boolean getBoolean(String arg) { + if (arg.equalsIgnoreCase("yes")) { + return true; + } else if (arg.equalsIgnoreCase("no")) { + return false; + } else { + return Boolean.parseBoolean(arg); + } + } + + public String getPriority() { + return priority; + } + + public SessionFactory getSessionFactory() { + return sessionFactory; + } + + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java new file mode 100644 index 00000000000..cc0d8f8c780 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java @@ -0,0 +1,76 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import com.yahoo.clientmetrics.MessageTypeMetricSet; +import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.concurrent.Timer; +import com.yahoo.metrics.Metric; +import com.yahoo.metrics.MetricSet; +import com.yahoo.metrics.MetricVisitor; + +import java.io.PrintStream; + +/** + * Class that takes progress from the feed and prints to a stream. + */ +public class BenchmarkProgressPrinter implements RouteMetricSet.ProgressCallback { + private final long startTime; + private final Timer timer; + private final PrintStream output; + + public BenchmarkProgressPrinter(Timer timer, PrintStream output) { + this.timer = timer; + this.output = output; + this.startTime = timer.milliTime(); + } + + class PrintVisitor extends MetricVisitor { + private final PrintStream out; + + PrintVisitor(PrintStream out) { + this.out = out; + } + + @Override + public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { + if (set instanceof MessageTypeMetricSet && set.getName().equals("total")) { + Metric m = set.getMetric("latency"); + Metric count = set.getMetric("count"); + Metric err = set.getMetric("errors.total"); + + long okCount = 0, errCount = 0, minLatency = 0, maxLatency = 0, avgLatency = 0; + + if (m != null) { + minLatency = m.getLongValue("min"); + maxLatency = m.getLongValue("max"); + avgLatency = m.getLongValue("average"); + } + if (count != null) { + okCount = count.getLongValue("count"); + } + + if (err != null) { + errCount = err.getLongValue("count"); + } + long timeUsed = timer.milliTime() - startTime; + out.println(timeUsed + ", " + okCount + ", " + errCount + ", " + minLatency + ", " + maxLatency + ", " + avgLatency); + } + return true; + } + } + + @Override + public void onProgress(RouteMetricSet metrics) { + //metrics.visit(new PrintVisitor(output), false); + } + + @Override + public void done(RouteMetricSet metrics) { + try { + output.println("# Time used, num ok, num error, min latency, max latency, average latency"); + metrics.visit(new PrintVisitor(output), false); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java new file mode 100755 index 00000000000..3479221257d --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java @@ -0,0 +1,14 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; + +public class FileRequest extends InputStreamRequest { + + FileRequest(File f) throws FileNotFoundException { + super(new FileInputStream(f)); + } + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java new file mode 100644 index 00000000000..e69eb6727b0 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java @@ -0,0 +1,38 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import com.yahoo.container.jdisc.HttpRequest; + +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +/** + * This is needed because whoever wrote this library moronically decided to pass in-process communication through + * the HTTP layer. As the feeded is being phased out in favor of the standalone HTTP client we don't bother to clean + * it up properly. + * + * @author bratseth + */ +public class InputStreamRequest { + + private InputStream input; + private Map properties = new HashMap<>(); + + protected InputStreamRequest(InputStream input) { + this.input = input; + } + + public void setProperty(String key, String value) { + properties.put(key, value); + } + + public String getProperty(String key) { + return properties.get(key); + } + + public HttpRequest toRequest() { + return HttpRequest.createTestRequest("", com.yahoo.jdisc.http.HttpRequest.Method.POST, input, properties); + } + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java new file mode 100644 index 00000000000..52087d33a47 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java @@ -0,0 +1,149 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import com.yahoo.clientmetrics.MessageTypeMetricSet; +import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.concurrent.Timer; +import com.yahoo.metrics.Metric; +import com.yahoo.metrics.MetricSet; +import com.yahoo.metrics.MetricVisitor; +import com.yahoo.metrics.SumMetric; + +import java.io.IOException; +import java.io.PrintStream; +import java.math.RoundingMode; +import java.text.NumberFormat; +import java.util.Locale; + +/** + * Class that takes progress from the feed and prints to a stream. + */ +public class ProgressPrinter implements RouteMetricSet.ProgressCallback { + private long startTime = 0; + private long lastProgressTime = 0; + private long lastVerboseProgress = 0; + final Timer timer; + final PrintStream output; + + public ProgressPrinter(Timer timer, PrintStream output) { + this.timer = timer; + this.output = output; + + startTime = timer.milliTime(); + lastProgressTime = startTime; + lastVerboseProgress = startTime; + } + + class PrintVisitor extends MetricVisitor { + final PrintStream out; + final NumberFormat format; + + PrintVisitor(PrintStream out) { + this.out = out; + format = NumberFormat.getNumberInstance(Locale.US); + format.setMaximumFractionDigits(2); + format.setMinimumFractionDigits(2); + format.setMinimumIntegerDigits(1); + format.setParseIntegerOnly(false); + format.setRoundingMode(RoundingMode.HALF_UP); + format.setGroupingUsed(false); + } + + @Override + public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { + if (set instanceof MessageTypeMetricSet && !set.getName().equals("total")) { + Metric m = set.getMetric("latency"); + Metric count = set.getMetric("count"); + Metric err = set.getMetric("errors.total"); + + long okCount = 0, errCount = 0, ignored = 0; + long minLatency = 0, maxLatency = 0, avgLatency = 0; + + if (m != null) { + minLatency = m.getLongValue("min"); + maxLatency = m.getLongValue("max"); + avgLatency = m.getLongValue("average"); + } + if (count != null) { + okCount = count.getLongValue("count"); + } + Metric ignoredMetric = set.getMetric("ignored"); + if (ignoredMetric != null) { + ignored = ignoredMetric.getLongValue("count"); + } + + if (err != null) { + errCount = err.getLongValue("count"); + } + + long timeSinceStart = timer.milliTime() - startTime; + + out.println(((MessageTypeMetricSet)set).getMessageName() + ":\t" + + "ok: " + okCount + + " msgs/sec: " + format.format((double)okCount * 1000 / timeSinceStart) + + " failed: " + errCount + + " ignored: " + ignored + + " latency(min, max, avg): " + minLatency + ", " + maxLatency + ", " + avgLatency); + } + return true; + } + } + + public static String getDashes(int count) { + String dashes = ""; + for (int i = 0; i < count; i++) { + dashes += "-"; + } + + return dashes; + } + + public synchronized void renderStatusText(RouteMetricSet metrics, PrintStream stream) throws IOException { + String headline = "Messages sent to vespa (route " + metrics.getName() + ") :"; + stream.println(headline); + stream.println(getDashes(headline.length())); + metrics.visit(new PrintVisitor(stream), false); + } + + public long getOkMessageCount(RouteMetricSet metrics) { + SumMetric sum = (SumMetric)metrics.getMetric("total"); + + MetricSet ms = (MetricSet)sum.generateSum(); + if (ms != null) { + Metric latency = ms.getMetric("latency"); + if (latency != null) { + return latency.getLongValue("count"); + } + } + + return 0; + } + + @Override + public void onProgress(RouteMetricSet metrics) { + try { + long timeNow = timer.milliTime(); + + if (timeNow - lastVerboseProgress > 30000) { + output.println("\n"); + renderStatusText(metrics, output); + lastVerboseProgress = timeNow; + } else if (timeNow - lastProgressTime > 1000) { + output.print("\rSuccessfully sent " + getOkMessageCount(metrics) + " messages so far"); + lastProgressTime = timeNow; + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Override + public void done(RouteMetricSet metrics) { + try { + output.println("\n"); + renderStatusText(metrics, output); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java new file mode 100755 index 00000000000..a6ede66c43d --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java @@ -0,0 +1,171 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespafeeder; + +import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; +import com.yahoo.feedapi.FeedContext; +import com.yahoo.feedhandler.FeedResponse; +import com.yahoo.feedhandler.NullFeedMetric; +import com.yahoo.feedhandler.VespaFeedHandler; +import com.yahoo.log.LogSetup; +import com.yahoo.concurrent.SystemTimer; +import com.yahoo.vespaclient.ClusterList; + +import java.io.*; +import java.util.*; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +public class VespaFeeder { + + Arguments args; + DocumentTypeManager manager; + Executor threadPool = Executors.newCachedThreadPool(ThreadFactoryFactory.getThreadFactory("vespafeeder")); + + public VespaFeeder(Arguments args, DocumentTypeManager manager) { + this.args = args; + this.manager = manager; + } + + public static class FeedErrorException extends Exception { + String message; + + public FeedErrorException(String message) { + this.message = message; + } + + @Override + public String getMessage() { + return message; + } + + } + + static FeedErrorException renderErrors(List errors) { + StringBuilder buffer = new StringBuilder(); + + if (!errors.isEmpty()) { + String headline = (errors.size() > 10) ? "First 10 errors (of " + errors.size() + "):" : "Errors:"; + buffer.append(headline).append("\n"); + for (int i = 0; i < headline.length(); ++i) { + buffer.append("-"); + } + buffer.append("\n"); + for (int i = 0; i < errors.size() && i < 10; ++i) { + buffer.append(" ").append(errors.get(i)).append("\n"); + } + } + + return new FeedErrorException(buffer.toString()); + } + + public RouteMetricSet.ProgressCallback createProgressCallback(PrintStream output) { + if ("benchmark".equals(args.getMode())) { + return new BenchmarkProgressPrinter(SystemTimer.INSTANCE, output); + } else { + return new ProgressPrinter(SystemTimer.INSTANCE, output); + } + } + + void parseFiles(InputStream stdin, PrintStream output) throws Exception { + FeedContext context = new FeedContext( + args.getPropertyProcessor(), + args.getSessionFactory(), + manager, + new ClusterList(), new NullFeedMetric()); + + final BufferedInputStream input = new BufferedInputStream(stdin); + VespaFeedHandler handler = VespaFeedHandler.createFromContext(context, threadPool); + + if (args.getFiles().isEmpty()) { + InputStreamRequest req = new InputStreamRequest(input); + setProperties(req, input); + FeedResponse response = (FeedResponse)handler.handle(req.toRequest(), createProgressCallback(output)); + if ( ! response.isSuccess()) { + throw renderErrors(response.getErrorList()); + } + } else { + if (args.isVerbose()) { + for (String fileName : args.getFiles()) { + long thisSize = new File(fileName).length(); + output.println("Size of file '" + fileName + "' is " + thisSize + " B."); + } + } + + for (String fileName : args.getFiles()) { + File f = new File(fileName); + FileRequest req = new FileRequest(f); + final BufferedInputStream inputSnooper = new BufferedInputStream(new FileInputStream(fileName)); + setProperties(req, inputSnooper); + inputSnooper.close(); + FeedResponse response = (FeedResponse)handler.handle(req.toRequest(), createProgressCallback(output)); + if (!response.isSuccess()) { + throw renderErrors(response.getErrorList()); + } + } + } + } + + // use BufferedInputStream to enforce the input.markSupported() == true + private void setProperties(InputStreamRequest req, BufferedInputStream input) throws IOException { + setPriority(req); + setCreateIfNonExistent(req); + setJsonInput(req, input); + } + + private void setPriority(InputStreamRequest req) { + if (args.getPriority() != null) { + req.setProperty("priority", args.getPriority()); + } + } + + private void setCreateIfNonExistent(InputStreamRequest req) { + if (args.getFeederConfig().createifnonexistent()) { + req.setProperty("createifnonexistent", "true"); + } + } + + // package access for easy testing + static void setJsonInput(InputStreamRequest req, BufferedInputStream input) throws IOException { + input.mark(4); + int b = input.read(); + input.reset(); + // A valid JSON feed will always start with '[' + if (b == '[') { + req.setProperty(VespaFeedHandler.JSON_INPUT, Boolean.TRUE.toString()); + } else { + req.setProperty(VespaFeedHandler.JSON_INPUT, Boolean.FALSE.toString()); + } + } + + public static void main(String[] args) { + LogSetup.initVespaLogging("vespafeeder"); + + try { + Arguments arguments = new Arguments(args, null); + + DocumentTypeManager manager = new DocumentTypeManager(); + DocumentTypeManagerConfigurer.configure(manager, "client").close(); + + VespaFeeder feeder = new VespaFeeder(arguments, manager); + feeder.parseFiles(System.in, System.out); + System.exit(0); + } catch (Arguments.HelpShownException e) { + System.exit(0); + } catch (IllegalArgumentException e) { + System.exit(1); + } catch (FileNotFoundException e) { + System.err.println("Could not open file " + e.getMessage()); + System.exit(1); + } catch (FeedErrorException e) { + System.err.println("\n" + e.getMessage()); + System.exit(1); + } catch (Exception e) { + System.err.println("Got exception " + e.getMessage() + ", aborting feed."); + System.exit(1); + } + } + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java new file mode 100644 index 00000000000..b57a9f7bf85 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java @@ -0,0 +1,160 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; + +import java.util.Iterator; +/** + * This class contains the the program parameters. + * + * @author bjorncs + */ +public class ClientParameters { + + // Determines if the help page should be presented + public final boolean help; + // Contains the document ids. Is backed by either a list iterator if the ids were given as CLI arguments or Scanner(System.in) if ids are provided by standard input. + public final Iterator documentIds; + // Print only the id for retrieved documents + public final boolean printIdsOnly; + // Determines which document fields to retrieve. Default is '[all]'. + public final String fieldSet; + // The Vespa route + public final String route; + // Alternative way to specify the route using cluster name. + public final String cluster; + // The configuration id for message bus. Default "client". + public final String configId; + // Determines if the serialized document size should be printed + public final boolean showDocSize; + // Document request timeout + public final double timeout; + // Determines whether or not the document request can be resent + public final boolean noRetry; + // Vespa trace level + public final int traceLevel; + // Document request priority + public final DocumentProtocol.Priority priority; + // Determines the Vespa load type + public final String loadTypeName; + // If full documents are printed, they will be printed as JSON (instead of XML) + public final boolean jsonOutput; + + + private ClientParameters( + boolean help, Iterator documentIds, boolean printIdsOnly, + String fieldSet, String route, String cluster, String configId, + boolean showDocSize, double timeout, boolean noRetry, int traceLevel, + DocumentProtocol.Priority priority, String loadTypeName, boolean jsonOutput) { + + this.help = help; + this.documentIds = documentIds; + this.printIdsOnly = printIdsOnly; + this.fieldSet = fieldSet; + this.route = route; + this.cluster = cluster; + this.configId = configId; + this.showDocSize = showDocSize; + this.timeout = timeout; + this.noRetry = noRetry; + this.traceLevel = traceLevel; + this.priority = priority; + this.loadTypeName = loadTypeName; + this.jsonOutput = jsonOutput; + } + + public static class Builder { + private boolean help; + private Iterator documentIds; + private boolean printIdsOnly; + private String fieldSet; + private String route; + private String cluster; + private String configId; + private boolean showDocSize; + private double timeout; + private boolean noRetry; + private int traceLevel; + private DocumentProtocol.Priority priority; + private String loadTypeName; + private boolean jsonOutput; + + public Builder setHelp(boolean help) { + this.help = help; + return this; + } + + public Builder setDocumentIds(Iterator documentIds) { + this.documentIds = documentIds; + return this; + } + + public Builder setPrintIdsOnly(boolean printIdsOnly) { + this.printIdsOnly = printIdsOnly; + return this; + } + + public Builder setFieldSet(String fieldSet) { + this.fieldSet = fieldSet; + return this; + } + + public Builder setRoute(String route) { + this.route = route; + return this; + } + + public Builder setCluster(String cluster) { + this.cluster = cluster; + return this; + } + + public Builder setConfigId(String configId) { + this.configId = configId; + return this; + } + + public Builder setShowDocSize(boolean showDocSize) { + this.showDocSize = showDocSize; + return this; + } + + public Builder setTimeout(double timeout) { + this.timeout = timeout; + return this; + } + + public Builder setNoRetry(boolean noRetry) { + this.noRetry = noRetry; + return this; + } + + public Builder setTraceLevel(int traceLevel) { + this.traceLevel = traceLevel; + return this; + } + + public Builder setPriority(DocumentProtocol.Priority priority) { + this.priority = priority; + return this; + } + + public Builder setLoadTypeName(String loadTypeName) { + this.loadTypeName = loadTypeName; + return this; + } + + public Builder setJsonOutput(boolean jsonOutput) { + this.jsonOutput = jsonOutput; + return this; + } + + public ClientParameters build() { + return new ClientParameters( + help, documentIds, printIdsOnly, fieldSet, route, cluster, configId, + showDocSize, timeout, noRetry, traceLevel, priority, loadTypeName, jsonOutput); + } + } + + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java new file mode 100644 index 00000000000..cbaef17a70a --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java @@ -0,0 +1,263 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +import java.io.InputStream; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Scanner; + +/** + * This class is responsible for parsing the command line arguments and print the help page. + * + * @author bjorncs + */ +public class CommandLineOptions { + + public static final String HELP_OPTION = "help"; + public static final String PRINTIDS_OPTION = "printids"; + public static final String HEADERSONLY_OPTION = "headersonly"; + public static final String FIELDSET_OPTION = "fieldset"; + public static final String CLUSTER_OPTION = "cluster"; + public static final String ROUTE_OPTION = "route"; + public static final String CONFIGID_OPTION = "configid"; + public static final String SHOWDOCSIZE_OPTION = "showdocsize"; + public static final String TIMEOUT_OPTION = "timeout"; + public static final String NORETRY_OPTION = "noretry"; + public static final String TRACE_OPTION = "trace"; + public static final String PRIORITY_OPTION = "priority"; + public static final String LOADTYPE_OPTION = "loadtype"; + public static final String JSONOUTPUT_OPTION = "jsonoutput"; + + private final Options options = createOptions(); + private final InputStream stdIn; + + public CommandLineOptions(InputStream stdIn) { + this.stdIn = stdIn; + } + + public CommandLineOptions() { + this(System.in); + } + + @SuppressWarnings("AccessStaticViaInstance") + private static Options createOptions() { + Options options = new Options(); + + options.addOption(Option.builder("h") + .hasArg(false) + .desc("Show this syntax page.") + .longOpt(HELP_OPTION) + .build()); + + options.addOption(Option.builder("i") + .hasArg(false) + .desc("Show only identifiers of retrieved documents.") + .longOpt(PRINTIDS_OPTION) + .build()); + + options.addOption(Option.builder("e") + .hasArg(false) + .desc("Retrieve header fields only. [Deprecated].") + .longOpt(HEADERSONLY_OPTION).build()); + + options.addOption(Option.builder("f") + .hasArg(true) + .desc("Retrieve the specified fields only (see http://vespa.corp.yahoo.com/5/documentation/reference/fieldsets.html) (default '[all]')") + .longOpt(FIELDSET_OPTION) + .argName("fieldset").build()); + + options.addOption(Option.builder("u") + .hasArg(true) + .desc("Send request to the given content cluster.") + .longOpt(CLUSTER_OPTION) + .argName("cluster").build()); + + options.addOption(Option.builder("r") + .hasArg(true) + .desc("Send request to the given messagebus route.") + .longOpt(ROUTE_OPTION) + .argName("route").build()); + + options.addOption(Option.builder("c") + .hasArg(true) + .desc("Use the specified config id for messagebus configuration.") + .longOpt(CONFIGID_OPTION) + .argName("configid").build()); + + options.addOption(Option.builder("s") + .hasArg(false) + .desc("Show binary size of document.") + .longOpt(SHOWDOCSIZE_OPTION).build()); + + options.addOption(Option.builder("t") + .hasArg(true) + .desc("Set timeout for the request in seconds (default 0).") + .longOpt(TIMEOUT_OPTION) + .argName("timeout") + .type(Number.class).build()); + + options.addOption(Option.builder("n") + .hasArg(false) + .desc("Do not retry operation on transient errors, as is default.") + .longOpt(NORETRY_OPTION).build()); + + options.addOption(Option.builder("a") + .hasArg(true) + .desc("Trace level to use (default 0).") + .longOpt(TRACE_OPTION) + .argName("trace") + .type(Number.class).build()); + + options.addOption(Option.builder("p") + .hasArg(true) + .desc("Priority (default 6).") + .longOpt(PRIORITY_OPTION) + .argName("priority") + .type(Number.class).build()); + + options.addOption(Option.builder("l") + .hasArg(true) + .desc("Load type (default \"\").") + .longOpt(LOADTYPE_OPTION) + .argName("loadtype").build()); + + options.addOption(Option.builder("j") + .hasArg(false) + .desc("JSON output") + .longOpt(JSONOUTPUT_OPTION).build()); + + return options; + } + + public void printHelp() { + HelpFormatter formatter = new HelpFormatter(); + + formatter.printHelp( + "vespaget [documentid...]", "Fetch a document from a Vespa Content cluster.", options, + "If one or more document identifier are specified, these documents will be " + + "retrieved. Otherwise, document identifiers (separated with line break) will be read from standard in.\n", + false); + } + + public ClientParameters parseCommandLineArguments(String[] args) throws IllegalArgumentException { + try { + CommandLineParser clp = new DefaultParser(); + CommandLine cl = clp.parse(options, args); + + boolean printIdsOnly = cl.hasOption(PRINTIDS_OPTION); + boolean headersOnly = cl.hasOption(HEADERSONLY_OPTION); + String fieldSet = cl.getOptionValue(FIELDSET_OPTION, ""); + String cluster = cl.getOptionValue(CLUSTER_OPTION, ""); + String route = cl.getOptionValue(ROUTE_OPTION, ""); + String configId = cl.getOptionValue(CONFIGID_OPTION, ""); + boolean help = cl.hasOption(HELP_OPTION); + String loadtype = cl.getOptionValue(LOADTYPE_OPTION, ""); + boolean noRetry = cl.hasOption(NORETRY_OPTION); + boolean showDocSize = cl.hasOption(SHOWDOCSIZE_OPTION); + boolean jsonOutput = cl.hasOption(JSONOUTPUT_OPTION); + int trace = getTrace(cl); + DocumentProtocol.Priority priority = getPriority(cl); + double timeout = getTimeout(cl); + Iterator documentIds = getDocumentIds(cl); + + if (printIdsOnly && headersOnly) { + throw new IllegalArgumentException("Print ids and headers only options are mutually exclusive."); + } + if ((printIdsOnly || headersOnly) && !fieldSet.isEmpty()) { + throw new IllegalArgumentException("Field set option can not be used in combination with print ids or headers only options."); + } + + if (printIdsOnly) { + fieldSet = "[id]"; + } else if (headersOnly) { + fieldSet = "[header]"; + } else if (fieldSet.isEmpty()) { + fieldSet = "[all]"; + } + + if (!cluster.isEmpty() && !route.isEmpty()) { + throw new IllegalArgumentException("Cluster and route options are mutually exclusive."); + } + + if (route.isEmpty() && cluster.isEmpty()) { + route = "default"; + } + + if (trace < 0 || trace > 9) { + throw new IllegalArgumentException("Invalid tracelevel: " + trace); + } + + if (configId.isEmpty()) { + configId = "client"; + } + + ClientParameters.Builder paramsBuilder = new ClientParameters.Builder(); + return paramsBuilder + .setDocumentIds(documentIds) + .setConfigId(configId) + .setFieldSet(fieldSet) + .setHelp(help) + .setPrintIdsOnly(printIdsOnly) + .setLoadTypeName(loadtype) + .setNoRetry(noRetry) + .setCluster(cluster) + .setRoute(route) + .setShowDocSize(showDocSize) + .setTraceLevel(trace) + .setPriority(priority) + .setTimeout(timeout) + .setJsonOutput(jsonOutput) + .build(); + } catch (ParseException pe) { + throw new IllegalArgumentException(pe.getMessage()); + } + } + + private Iterator getDocumentIds(CommandLine cl) { + // Fetch document ids from stdin if no ids are passed in as command line arguments + List documentIds = Arrays.asList(cl.getArgs()); + // WARNING: CommandLine.getArgs may return a single empty string as the only element + if (documentIds.isEmpty() || + documentIds.size() == 1 && documentIds.get(0).isEmpty()) { + return new Scanner(stdIn); + } else { + return documentIds.iterator(); + } + } + + private static double getTimeout(CommandLine cl) throws ParseException { + Number timeoutObj = (Number) cl.getParsedOptionValue(TIMEOUT_OPTION); + return timeoutObj != null ? timeoutObj.doubleValue() : 0; + } + + private static int getTrace(CommandLine cl) throws ParseException { + Number traceObj = (Number) cl.getParsedOptionValue(TRACE_OPTION); + return traceObj != null ? traceObj.intValue() : 0; + } + + private static DocumentProtocol.Priority getPriority(CommandLine cl) throws ParseException { + Number priorityObj = (Number) cl.getParsedOptionValue(PRIORITY_OPTION); + int priorityNumber = priorityObj != null ? priorityObj.intValue() : DocumentProtocol.Priority.NORMAL_2.getValue(); + return parsePriority(priorityNumber); + } + + private static DocumentProtocol.Priority parsePriority(int n) { + for (DocumentProtocol.Priority priority : DocumentProtocol.Priority.values()) { + if (priority.getValue() == n) { + return priority; + } + } + throw new IllegalArgumentException("Invalid priority: " + n); + } + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentAccessFactory.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentAccessFactory.java new file mode 100644 index 00000000000..6836f033c11 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentAccessFactory.java @@ -0,0 +1,17 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; + +/** + * Factory class for {@link com.yahoo.documentapi.messagebus.MessageBusDocumentAccess}. + * + * @author bjorncs + */ +public class DocumentAccessFactory { + + public MessageBusDocumentAccess createDocumentAccess(MessageBusParams messageBusParams) { + return new MessageBusDocumentAccess(messageBusParams); + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java new file mode 100644 index 00000000000..6e52e89c580 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java @@ -0,0 +1,207 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.json.JsonWriter; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.MessageBusSyncSession; +import com.yahoo.documentapi.messagebus.loadtypes.LoadType; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.Trace; +import com.yahoo.text.Utf8; +import com.yahoo.vespaclient.ClusterDef; +import com.yahoo.vespaclient.ClusterList; + +import java.util.Iterator; +import java.util.Map; + +/** + * The document retriever is responsible for retrieving documents using the Document API and printing the result to standard out. + * + * @author bjorncs + */ +public class DocumentRetriever { + + private final ClusterList clusterList; + private final DocumentAccessFactory documentAccessFactory; + private final ClientParameters params; + private final LoadTypeSet loadTypeSet; + + private MessageBusSyncSession session; + private MessageBusDocumentAccess documentAccess; + + public DocumentRetriever(ClusterList clusterList, + DocumentAccessFactory documentAccessFactory, + LoadTypeSet loadTypeSet, + ClientParameters params) { + this.clusterList = clusterList; + this.documentAccessFactory = documentAccessFactory; + this.loadTypeSet = loadTypeSet; + this.params = params; + } + + public void shutdown() { + try { + if (session != null) { + session.destroy(); + } + } catch (IllegalStateException e) { + // Ignore exception on shutdown + } + try { + if (documentAccess != null) { + documentAccess.shutdown(); + } + } catch (IllegalStateException e) { + // Ignore exception on shutdown + } + } + + public void retrieveDocuments() throws DocumentRetrieverException { + boolean first = true; + String route = params.cluster.isEmpty() ? params.route : resolveClusterRoute(params.cluster); + LoadType loadType = params.loadTypeName.isEmpty() ? null : resolveLoadType(params.loadTypeName); + + MessageBusParams messageBusParams = createMessageBusParams(params.configId, params.timeout, route); + documentAccess = documentAccessFactory.createDocumentAccess(messageBusParams); + session = documentAccess.createSyncSession(new SyncParameters()); + int trace = params.traceLevel; + if (trace > 0) { + session.setTraceLevel(trace); + } + + Iterator iter = params.documentIds; + if (params.jsonOutput && !params.printIdsOnly) { + System.out.println('['); + } + while (iter.hasNext()) { + if (params.jsonOutput && !params.printIdsOnly) { + if (!first) { + System.out.println(','); + } else { + first = false; + } + } + String docid = iter.next(); + Message msg = createDocumentRequest(docid, loadType); + Reply reply = session.syncSend(msg); + printReply(reply); + } + if (params.jsonOutput && !params.printIdsOnly) { + System.out.println(']'); + } + } + + private String resolveClusterRoute(String clusterName) throws DocumentRetrieverException { + if (clusterList.getStorageClusters().isEmpty()) { + throw new DocumentRetrieverException("The Vespa cluster does not have any content clusters declared."); + } + + ClusterDef clusterDef = null; + for (ClusterDef c : clusterList.getStorageClusters()) { + if (c.getName().equals(clusterName)) { + clusterDef = c; + } + } + if (clusterDef == null) { + String names = createClusterNamesString(); + throw new DocumentRetrieverException(String.format( + "The Vespa cluster contains the content clusters %s, not %s. Please select a valid vespa cluster.", + names, clusterName)); + } + return String.format("[Storage:cluster=%s;clusterconfigid=%s]", clusterDef.getName(), clusterDef.getConfigId()); + } + + private LoadType resolveLoadType(String loadTypeName) throws DocumentRetrieverException { + Map loadTypesNameMap = loadTypeSet.getNameMap(); + if (!loadTypesNameMap.containsKey(loadTypeName)) { + throw new DocumentRetrieverException(String.format("Loadtype with name '%s' does not exist.\n", loadTypeName)); + } else { + return loadTypesNameMap.get(loadTypeName); + } + } + + private MessageBusParams createMessageBusParams(String configId, double timeout, String route) { + MessageBusParams messageBusParams = new MessageBusParams(loadTypeSet); + messageBusParams.setRoute(route); + messageBusParams.setProtocolConfigId(configId); + messageBusParams.setRoutingConfigId(configId); + messageBusParams.setDocumentManagerConfigId(configId); + + if (timeout > 0) { + messageBusParams.getSourceSessionParams().setTimeout(timeout); + } + return messageBusParams; + } + + private Message createDocumentRequest(String docid, LoadType loadType) { + GetDocumentMessage msg = new GetDocumentMessage(new DocumentId(docid), params.fieldSet); + msg.setPriority(params.priority); + msg.setRetryEnabled(!params.noRetry); + + if (loadType != null) { + msg.setLoadType(loadType); + } + return msg; + } + + private void printReply(Reply reply) { + Trace trace = reply.getTrace(); + if (!trace.getRoot().isEmpty()) { + System.out.println(trace); + } + + if (reply.hasErrors()) { + System.err.print("Request failed: "); + for (int i = 0; i < reply.getNumErrors(); i++) { + System.err.printf("\n %s", reply.getError(i)); + } + System.err.println(); + return; + } + + if (!(reply instanceof GetDocumentReply)) { + System.err.printf("Unexpected reply %s: '%s'\n", reply.getType(), reply.toString()); + return; + } + + GetDocumentReply documentReply = (GetDocumentReply) reply; + Document document = documentReply.getDocument(); + + if (document == null) { + System.out.println("Document not found."); + return; + } + + if (params.showDocSize) { + System.out.printf("Document size: %d bytes.\n", document.getSerializedSize()); + } + if (params.printIdsOnly) { + System.out.println(document.getId()); + } else { + if (params.jsonOutput) { + System.out.print(Utf8.toString(JsonWriter.toByteArray(document))); + } else { + System.out.print(document.toXML(" ")); + } + } + } + + private String createClusterNamesString() { + StringBuilder names = new StringBuilder(); + for (ClusterDef c : clusterList.getStorageClusters()) { + if (names.length() > 0) { + names.append(", "); + } + names.append(c.getName()); + } + return names.toString(); + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetrieverException.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetrieverException.java new file mode 100644 index 00000000000..4cf0e9885a3 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetrieverException.java @@ -0,0 +1,14 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + +/** + * Exception thrown by {@link DocumentRetriever}. + * + * @author bjorncs + */ +public class DocumentRetrieverException extends Exception { + + public DocumentRetrieverException(String message) { + super(message); + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java new file mode 100644 index 00000000000..324107d8909 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java @@ -0,0 +1,46 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaget; + + +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.vespaclient.ClusterList; + +/** + * The vespaget tool retrieves documents from a Vespa Document Storage cluster, and prints them to stdout as XML. + * + * @author bjorncs + */ +public class Main { + + public static void main(String[] args) { + try { + CommandLineOptions options = new CommandLineOptions(); + ClientParameters params = options.parseCommandLineArguments(args); + + if (params.help) { + options.printHelp(); + } else { + DocumentRetriever documentRetriever = createDocumentRetriever(params); + addShutdownHook(documentRetriever); + documentRetriever.retrieveDocuments(); + } + } catch (IllegalArgumentException e) { + System.err.printf("Failed to parse command line arguments: %s.\n", e.getMessage()); + } catch (DocumentRetrieverException e) { + System.err.printf("Failed to retrieve documents: %s\n", e.getMessage()); + } + } + + private static void addShutdownHook(DocumentRetriever documentRetriever) { + Runtime.getRuntime().addShutdownHook(new Thread(documentRetriever::shutdown)); + } + + private static DocumentRetriever createDocumentRetriever(ClientParameters params) { + return new DocumentRetriever( + new ClusterList("client"), + new DocumentAccessFactory(), + new LoadTypeSet(params.configId), + params + ); + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsException.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsException.java new file mode 100644 index 00000000000..a6f471e7b5e --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsException.java @@ -0,0 +1,18 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +/** + * Exception class used by {@link com.yahoo.vespastat.BucketStatsRetriever}. + * + * @author bjorncs + */ +public class BucketStatsException extends Exception { + public BucketStatsException(String message) { + super(message); + } + + public BucketStatsException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsPrinter.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsPrinter.java new file mode 100644 index 00000000000..a4a263188b5 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsPrinter.java @@ -0,0 +1,59 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +import com.yahoo.document.BucketId; +import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply; + +import java.io.PrintStream; +import java.util.List; + +/** + * The class is responsible for printing bucket information to a printstream. + * + * @author bjorncs + */ +public class BucketStatsPrinter { + private final BucketStatsRetriever retriever; + private final PrintStream out; + + public BucketStatsPrinter( + BucketStatsRetriever retriever, + PrintStream out) { + this.retriever = retriever; + this.out = out; + } + + public void retrieveAndPrintBucketStats(ClientParameters.SelectionType type, String id, boolean dumpData) throws BucketStatsException { + BucketId bucketId = retriever.getBucketIdForType(type, id); + if (type == ClientParameters.SelectionType.GROUP || type == ClientParameters.SelectionType.USER) { + out.printf("Generated 32-bit bucket id: %s\n", bucketId); + } + + List bucketList = retriever.retrieveBucketList(bucketId); + printBucketList(bucketList); + + if (dumpData) { + for (GetBucketListReply.BucketInfo bucketInfo : bucketList) { + BucketId bucket = bucketInfo.getBucketId(); + String bucketStats = retriever.retrieveBucketStats(type, id, bucket); + printBucketStats(bucket, bucketStats); + } + } + } + + private void printBucketList(List bucketList) { + if (bucketList.isEmpty()) { + out.println("No actual files were stored for this bucket."); + } else { + out.println("Bucket maps to the following actual files:"); + for (GetBucketListReply.BucketInfo bucketInfo : bucketList) { + out.printf("\t%s\n", bucketInfo); + } + } + } + + private void printBucketStats(BucketId bucket, String stats) { + out.printf("\nDetails for %s:\n%s", bucket, stats); + } + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java new file mode 100644 index 00000000000..84e89349f9f --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/BucketStatsRetriever.java @@ -0,0 +1,176 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +import com.yahoo.document.BucketId; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.DocumentId; +import com.yahoo.document.GlobalId; +import com.yahoo.document.select.BucketSelector; +import com.yahoo.document.select.BucketSet; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusSyncSession; +import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetBucketListMessage; +import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply; +import com.yahoo.documentapi.messagebus.protocol.StatBucketMessage; +import com.yahoo.documentapi.messagebus.protocol.StatBucketReply; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.routing.Route; + +import java.util.List; + +/** + * This class fetches bucket information from Vespa + * + * @author bjorncs + */ +public class BucketStatsRetriever { + + private final BucketIdFactory bucketIdFactory = new BucketIdFactory(); + private final BucketSelector selector = new BucketSelector(bucketIdFactory); + + private final MessageBusSyncSession session; + private final MessageBusDocumentAccess documentAccess; + private final String route; + + public BucketStatsRetriever( + DocumentAccessFactory documentAccessFactory, + String route, + ShutdownHookRegistrar registrar) { + registerShutdownHook(registrar); + this.documentAccess = documentAccessFactory.createDocumentAccess(); + this.session = documentAccess.createSyncSession(new SyncParameters()); + this.route = route; + } + + private void registerShutdownHook(ShutdownHookRegistrar registrar) { + registrar.registerShutdownHook(() -> { + try { + session.destroy(); + } catch (Exception e) { + // Ignore exception on shutdown + } + try { + documentAccess.shutdown(); + } catch (Exception e) { + // Ignore exception on shutdown + } + }); + } + + public BucketId getBucketIdForType(ClientParameters.SelectionType type, String id) throws BucketStatsException { + switch (type) { + case DOCUMENT: + return bucketIdFactory.getBucketId(new DocumentId(id)); + case BUCKET: + // The internal parser of BucketID is used since the Java Long.decode cannot handle unsigned longs. + return new BucketId(String.format("BucketId(%s)", id)); + case GID: + return convertGidToBucketId(id); + case USER: + case GROUP: + try { + BucketSet bucketList = selector.getBucketList(createDocumentSelection(type, id)); + if (bucketList.size() != 1) { + String message = String.format("Document selection must map to only one location. " + + "Specified selection matches %d locations.", bucketList.size()); + throw new BucketStatsException(message); + } + return bucketList.iterator().next(); + } catch (ParseException e) { + throw new BucketStatsException(String.format("Invalid id: %s (%s).", id, e.getMessage()), e); + } + default: + throw new RuntimeException("Unreachable code"); + } + } + + public String retrieveBucketStats(ClientParameters.SelectionType type, String id, BucketId bucketId) throws BucketStatsException { + String documentSelection = createDocumentSelection(type, id); + StatBucketMessage msg = new StatBucketMessage(bucketId, documentSelection); + StatBucketReply statBucketReply = sendMessage(msg, StatBucketReply.class); + return statBucketReply.getResults(); + } + + public List retrieveBucketList(BucketId bucketId) throws BucketStatsException { + GetBucketListMessage msg = new GetBucketListMessage(bucketId); + GetBucketListReply bucketListReply = sendMessage(msg, GetBucketListReply.class); + return bucketListReply.getBuckets(); + } + + + private T sendMessage(DocumentMessage msg, Class expectedReply) throws BucketStatsException { + setRoute(msg, route); + Reply reply = session.syncSend(msg); + return validateReply(reply, expectedReply); + } + + private static void setRoute(DocumentMessage msg, String route) throws BucketStatsException { + try { + msg.setRoute(Route.parse(route)); + } catch (Exception e) { + throw new BucketStatsException(String.format("Invalid route: '%s'.", route)); + } + } + + private static T validateReply(Reply reply, Class type) throws BucketStatsException { + if (reply.hasErrors()) { + throw new BucketStatsException(makeErrorMessage(reply)); + } + if (!type.isInstance(reply)) { + throw new BucketStatsException(String.format("Unexpected reply %s: '%s'", reply.getType(), reply.toString())); + } + return type.cast(reply); + } + + private static String makeErrorMessage(Reply reply) { + StringBuilder b = new StringBuilder(); + b.append("Request failed: \n"); + for (int i = 0; i < reply.getNumErrors(); i++) { + b.append(String.format("\t %s\n", reply.getError(i))); + } + return b.toString(); + } + + private static String createDocumentSelection(ClientParameters.SelectionType type, String id) { + switch (type) { + case BUCKET: + return "true"; + case DOCUMENT: + return String.format("id=\"%s\"", id); + case GID: + return String.format("id.gid=\"gid(%s)\"", id); + case USER: + return String.format("id.user=%s", id); + case GROUP: + return String.format("id.group=\"%s\"", id); + default: + throw new RuntimeException("Unreachable code"); + } + } + + private static BucketId convertGidToBucketId(String id) throws BucketStatsException { + if (!id.matches("0x\\p{XDigit}{24}")) { + throw new BucketStatsException("Invalid gid: " + id); + } + String hexWithoutPrefix = id.substring(2); + return new GlobalId(convertHexStringToByteArray(hexWithoutPrefix)).toBucketId(); + } + + private static byte[] convertHexStringToByteArray(String s) throws BucketStatsException { + int len = s.length(); + byte[] data = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + int digit1 = Character.digit(s.charAt(i), 16); + int digit2 = Character.digit(s.charAt(i + 1), 16); + data[i / 2] = (byte) ((digit1 << 4) + digit2); + } + return data; + } + + public interface ShutdownHookRegistrar { + void registerShutdownHook(Runnable runnable); + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/ClientParameters.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/ClientParameters.java new file mode 100644 index 00000000000..bba0d9803ed --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/ClientParameters.java @@ -0,0 +1,73 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +/** + * This class contains the program parameters. + * + * @author bjorncs + */ +public class ClientParameters { + // Show help page if true + public final boolean help; + // Dump list of documents for all buckets matching the selection if true + public final boolean dumpData; + // The message bus route + public final String route; + // The selection type + public final SelectionType selectionType; + // The selection id + public final String id; + + public ClientParameters( + boolean help, + boolean dumpData, + String route, + SelectionType selectionType, + String id) { + this.help = help; + this.dumpData = dumpData; + this.route = route; + this.selectionType = selectionType; + this.id = id; + } + + public enum SelectionType {USER, GROUP, BUCKET, GID, DOCUMENT} + + public static class Builder { + private boolean help; + private boolean dumpData; + private String route; + private SelectionType selectionType; + private String id; + + public Builder setHelp(boolean help) { + this.help = help; + return this; + } + + public Builder setDumpData(boolean dumpData) { + this.dumpData = dumpData; + return this; + } + + public Builder setRoute(String route) { + this.route = route; + return this; + } + + public Builder setSelectionType(SelectionType selectionType) { + this.selectionType = selectionType; + return this; + } + + public Builder setId(String id) { + this.id = id; + return this; + } + + public ClientParameters build() { + return new ClientParameters(help, dumpData, route, selectionType, id); + } + } + +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/CommandLineOptions.java new file mode 100644 index 00000000000..b0b6246a262 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/CommandLineOptions.java @@ -0,0 +1,139 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +/** + * Responsible for parsing the command line arguments and presenting the help page + * + * @author bjorncs + */ +public class CommandLineOptions { + + private static final String HELP_OPTION = "help"; + private static final String DUMP_OPTION = "dump"; + private static final String ROUTE_OPTION = "route"; + private static final String USER_OPTION = "user"; + private static final String GROUP_OPTION = "group"; + private static final String BUCKET_OPTION = "bucket"; + private static final String GID_OPTION = "gid"; + private static final String DOCUMENT_OPTION = "document"; + + private final Options options = createOptions(); + + @SuppressWarnings("AccessStaticViaInstance") + private static Options createOptions() { + Options options = new Options(); + + options.addOption(Option.builder("h") + .hasArg(false) + .desc("Show this syntax page.") + .longOpt(HELP_OPTION) + .build()); + + options.addOption(Option.builder("d") + .hasArg(false) + .desc("Dump list of documents for all buckets matching the selection command.") + .longOpt(DUMP_OPTION) + .build()); + + options.addOption(Option.builder("r") + .hasArg(true) + .desc("Route to send the messages to, usually the name of the storage cluster.") + .argName("route") + .longOpt(ROUTE_OPTION) + .build()); + + // A group of mutually exclusive options for user, group, bucket, gid and document. + OptionGroup optionGroup = new OptionGroup(); + optionGroup.setRequired(false); + + optionGroup.addOption(Option.builder("u") + .hasArg(true) + .desc("Dump list of buckets that can contain the given user.") + .argName("userid") + .longOpt(USER_OPTION) + .build()); + + optionGroup.addOption(Option.builder("g") + .hasArg(true) + .desc("Dump list of buckets that can contain the given group.") + .argName("groupid") + .longOpt(GROUP_OPTION) + .build()); + + optionGroup.addOption(Option.builder("b") + .hasArg(true) + .desc("Dump list of buckets that are contained in the given bucket, or that contain it.") + .argName("bucketid") + .longOpt(BUCKET_OPTION) + .build()); + + optionGroup.addOption(Option.builder("l") + .hasArg(true) + .desc("Dump information about one specific document, as given by the GID (implies --dump).") + .argName("globalid") + .longOpt(GID_OPTION) + .build()); + + optionGroup.addOption(Option.builder("o") + .hasArg(true) + .desc("Dump information about one specific document (implies --dump).") + .argName("docid") + .longOpt(DOCUMENT_OPTION) + .build()); + + options.addOptionGroup(optionGroup); + return options; + } + + public void printHelp() { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("vdsstat [options]", + "Fetch statistics about a specific user, group, bucket, gid or document.", options, "", false); + } + + public ClientParameters parseCommandLineArguments(String[] args) { + try { + CommandLineParser clp = new DefaultParser(); + CommandLine cl = clp.parse(options, args); + ClientParameters.Builder builder = new ClientParameters.Builder(); + + builder.setHelp(cl.hasOption(HELP_OPTION)); + builder.setDumpData(cl.hasOption(DUMP_OPTION)); + builder.setRoute(cl.getOptionValue(ROUTE_OPTION, "default")); + + if (cl.hasOption(USER_OPTION)) { + builder.setSelectionType(ClientParameters.SelectionType.USER); + builder.setId(cl.getOptionValue(USER_OPTION)); + } else if (cl.hasOption(GROUP_OPTION)) { + builder.setSelectionType(ClientParameters.SelectionType.GROUP); + builder.setId(cl.getOptionValue(GROUP_OPTION)); + } else if (cl.hasOption(BUCKET_OPTION)) { + builder.setSelectionType(ClientParameters.SelectionType.BUCKET); + builder.setId(cl.getOptionValue(BUCKET_OPTION)); + } else if (cl.hasOption(GID_OPTION)) { + builder.setSelectionType(ClientParameters.SelectionType.GID); + builder.setId(cl.getOptionValue(GID_OPTION)); + builder.setDumpData(true); + } else if (cl.hasOption(DOCUMENT_OPTION)) { + builder.setSelectionType(ClientParameters.SelectionType.DOCUMENT); + builder.setId(cl.getOptionValue(DOCUMENT_OPTION)); + builder.setDumpData(true); + } else if (!cl.hasOption(HELP_OPTION)) { + throw new IllegalArgumentException("Must specify one of 'user', 'group', 'bucket', 'document' or 'gid'."); + } + + return builder.build(); + } catch (ParseException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/DocumentAccessFactory.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/DocumentAccessFactory.java new file mode 100644 index 00000000000..55a31f30a2b --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/DocumentAccessFactory.java @@ -0,0 +1,15 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; + +/** + * Factory class for {@link com.yahoo.documentapi.messagebus.MessageBusDocumentAccess}. + * + * @author bjorncs + */ +public class DocumentAccessFactory { + public MessageBusDocumentAccess createDocumentAccess() { + return new MessageBusDocumentAccess(); + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java new file mode 100644 index 00000000000..9d87a6f68f4 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespastat/Main.java @@ -0,0 +1,38 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespastat; + +/** + * Main application class + * + * @author bjorncs + */ +public class Main { + + private Main() { + } + + public static void main(String[] args) { + CommandLineOptions options = new CommandLineOptions(); + try { + ClientParameters params = options.parseCommandLineArguments(args); + if (params.help) { + options.printHelp(); + return; + } + BucketStatsRetriever retriever = new BucketStatsRetriever( + new DocumentAccessFactory(), + params.route, + createShutdownHookRegistrar()); + BucketStatsPrinter printer = new BucketStatsPrinter(retriever, System.out); + printer.retrieveAndPrintBucketStats(params.selectionType, params.id, params.dumpData); + } catch (IllegalArgumentException e) { + System.err.printf("Failed to parse command line arguments: %s.\n", e.getMessage()); + } catch (BucketStatsException e) { + System.err.println(e.getMessage()); + } + } + + private static BucketStatsRetriever.ShutdownHookRegistrar createShutdownHookRegistrar() { + return runnable -> Runtime.getRuntime().addShutdownHook(new Thread(runnable)); + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java b/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java new file mode 100644 index 00000000000..803445d16f5 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java @@ -0,0 +1,162 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespasummarybenchmark; + +import com.yahoo.compress.CompressionType; +import com.yahoo.document.GlobalId; +import com.yahoo.document.idstring.IdString; +import com.yahoo.document.serialization.DeserializationException; +import com.yahoo.jrt.*; +import com.yahoo.log.LogSetup; +import com.yahoo.slime.*; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; + +/** + * + * This is used for testing and benchmarking rpc docsum interface. + * time vespa-summary-benchmark file-containing-docids connectionspec summary-class repetitions threads + * fx ' time vespa-summary-benchmark feed.xml tcp/localhost:19115 keyvaluesummary 10000 32' + * + * @author baldersheim + */ +public class VespaSummaryBenchmark { + + private final Supervisor supervisor = new Supervisor(new Transport()); + + private VespaSummaryBenchmark() { } + + private static List getDocIds(String fileName) { + try { + FileInputStream fstream = new FileInputStream(fileName); + DataInputStream in = new DataInputStream(fstream); + BufferedReader br = new BufferedReader(new InputStreamReader(in)); + String strLine; + + List docIds = new ArrayList<>(); + while ((strLine = br.readLine()) != null) { + docIds.add(strLine); + } + in.close(); + return docIds; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private List getTargets(String connectionSpec, int numTargets) { + List targets = new ArrayList<>(numTargets); + for ( int i=0; i < numTargets; i++) { + targets.add(supervisor.connect(new Spec(connectionSpec))); + } + return targets; + } + + private static Slime createDocsumRequest(String summaryClass, List gids) { + Slime docsumRequest = new Slime(); + Cursor root = docsumRequest.setObject(); + root.setString("class", summaryClass); + Cursor gidCursor = root.setArray("gids"); + for (GlobalId gid : gids) { + gidCursor.addData(gid.getRawId()); + } + return docsumRequest; + } + + private static class Waiter implements RequestWaiter { + + int waitingFor; + boolean dump; + + Waiter(int expect, boolean dump) { + waitingFor = expect; + this.dump = dump; + } + + private void print(Request request) { + Values ret = request.returnValues(); + CompressionType type = CompressionType.valueOf(ret.get(0).asInt8()); + int uncompressedSize = ret.get(1).asInt32(); + byte [] blob = ret.get(2).asData(); + if (type == CompressionType.LZ4) { + LZ4Factory factory = LZ4Factory.fastestInstance(); + LZ4FastDecompressor decompressor = factory.fastDecompressor(); + byte [] uncompressed = new byte [uncompressedSize]; + int compressedLength = decompressor.decompress(blob, 0, uncompressed, 0, uncompressedSize); + if (compressedLength != blob.length) { + throw new DeserializationException("LZ4 decompression failed. compressed size does not match. Expected " + blob.length + ". Got " + compressedLength); + } + blob = uncompressed; + } + Slime slime = BinaryFormat.decode(blob); + try { + new JsonFormat(true).encode(System.out, slime); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void handleRequestDone(Request request) { + synchronized (this) { + if (dump) { + print(request); + dump = false; + } + waitingFor--; + if (waitingFor == 0) { + this.notifyAll(); + } + } + } + void waitForReplies() throws InterruptedException { + synchronized (this) { + while (waitingFor > 0) { + this.wait(); + } + } + } + } + + private static void fetchDocIds(String summaryClass, List targets, List gids, boolean dump) { + Slime docsumRequest = createDocsumRequest(summaryClass, gids); + byte [] blob = BinaryFormat.encode(docsumRequest); + Waiter waiter = new Waiter(targets.size(), dump); + for (Target target : targets) { + Request r = new Request("proton.getDocsums"); + r.parameters().add(new Int8Value(CompressionType.NONE.getCode())); + r.parameters().add(new Int32Value(blob.length)); + r.parameters().add(new DataValue(blob)); + target.invokeAsync(r, 100.0, waiter); + } + try { + waiter.waitForReplies(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + } + + public static void main(String[] args) { + LogSetup.initVespaLogging("vespasummarybenchmark"); + String docidFileName = args[0]; + String connectionSpec = args[1]; + String summaryClass = args[2]; + int numRuns = Integer.parseInt(args[3]); + int numTargets = Integer.parseInt(args[4]); + VespaSummaryBenchmark benchmark = new VespaSummaryBenchmark(); + List docIds = getDocIds(docidFileName); + List gids = new ArrayList<>(docIds.size()); + for (String docid : docIds) { + GlobalId gid = new GlobalId(IdString.createIdString(docid)); + gids.add(gid); + } + List targets = benchmark.getTargets(connectionSpec, numTargets); + for (int i = 0; i < numRuns; i++) { + fetchDocIds(summaryClass, targets, gids, i==0); + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java new file mode 100644 index 00000000000..c849bc1741c --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java @@ -0,0 +1,292 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespavisit; + +import com.yahoo.document.BucketId; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.json.JsonWriter; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.documentapi.AckToken; +import com.yahoo.documentapi.DumpVisitorDataHandler; +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.documentapi.VisitorControlHandler; +import com.yahoo.documentapi.VisitorDataHandler; +import com.yahoo.documentapi.messagebus.protocol.DocumentListEntry; +import com.yahoo.documentapi.messagebus.protocol.DocumentListMessage; +import com.yahoo.documentapi.messagebus.protocol.EmptyBucketsMessage; +import com.yahoo.documentapi.messagebus.protocol.MapVisitorMessage; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.Message; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +/** + * A visitor data and progress handler that writes to STDOUT. + * + * Due to java not being able to inherit two classes, and neither being an + * interface this had to be implemented by creating a wrapper class. + * + * @author Thomas Gundersen + */ +public class StdOutVisitorHandler extends VdsVisitHandler { + private static final Logger log = Logger.getLogger( + StdOutVisitorHandler.class.getName()); + private boolean printIds; + private boolean indentXml; + private int processTimeMilliSecs; + private PrintStream out; + private final boolean jsonOutput; + + private VisitorDataHandler dataHandler; + + public StdOutVisitorHandler(boolean printIds, boolean indentXml, + boolean showProgress, boolean showStatistics, boolean doStatistics, + boolean abortOnClusterDown, int processtime, boolean jsonOutput) + { + super(showProgress, showStatistics, abortOnClusterDown); + + this.printIds = printIds; + this.indentXml = indentXml; + this.processTimeMilliSecs = processtime; + this.jsonOutput = jsonOutput; + String charset = "UTF-8"; + try { + out = new PrintStream(System.out, true, charset); + } catch (java.io.UnsupportedEncodingException e) { + System.out.println(charset + " is an unsupported encoding, " + + "using default instead."); + out = System.out; + } + + dataHandler = new DataHandler(doStatistics); + } + + @Override + public void onDone() { + } + + public VisitorDataHandler getDataHandler() { return dataHandler; } + + class StatisticsMap extends LinkedHashMap { + int maxSize; + + StatisticsMap(int maxSize) { + super(100, (float)0.75, true); + this.maxSize = maxSize; + } + + protected boolean removeEldestEntry(Map.Entry eldest) { + if (size() > maxSize) { + dump(eldest); + return true; + } + + return false; + } + + private void dump(Map.Entry e) { + out.println(e.getKey() + ":" + e.getValue()); + } + + public void dumpAll() { + for (Map.Entry e : entrySet()) { + dump(e); + } + clear(); + } + } + + class DataHandler extends DumpVisitorDataHandler { + boolean doStatistics; + StatisticsMap statisticsMap = new StatisticsMap(10000); + private volatile boolean first = true; + + public DataHandler(boolean doStatistics) { + this.doStatistics = doStatistics; + } + + @Override + public void onMessage(Message m, AckToken token) { + if (processTimeMilliSecs > 0) { + try { + Thread.sleep(processTimeMilliSecs); + } catch (InterruptedException e) {} + } + + synchronized (printLock) { + if (m instanceof MapVisitorMessage) { + onMapVisitorData(((MapVisitorMessage)m).getData()); + ack(token); + } else if (m instanceof DocumentListMessage) { + DocumentListMessage dlm = (DocumentListMessage)m; + onDocumentList(dlm.getBucketId(), dlm.getDocuments()); + ack(token); + } else if (m instanceof EmptyBucketsMessage) { + onEmptyBuckets(((EmptyBucketsMessage)m).getBucketIds()); + ack(token); + } else { + super.onMessage(m, token); + } + } + } + + @Override + public void onDocument(Document doc, long timestamp) { + try { + if (lastLineIsProgress) { + System.err.print('\r'); + } + + if (printIds) { + out.print(doc.getId()); + out.print(" (Last modified at "); + out.println(timestamp + ")"); + } else { + if (jsonOutput) { + writeJsonDocument(doc); + } else { + out.print(doc.toXML( + indentXml ? " " : "")); + } + } + } catch (Exception e) { + System.err.println("Failed to output document: " + + e.getMessage()); + getControlHandler().abort(); + } + } + + private void writeJsonDocument(Document doc) throws IOException { + writeFeedStartOrRecordSeparator(); + out.write(JsonWriter.toByteArray(doc)); + } + + @Override + public void onRemove(DocumentId docId) { + try { + if (lastLineIsProgress) { + System.err.print('\r'); + } + + if (printIds) { + out.println(docId + " (Removed)"); + } else { + if (jsonOutput) { + writeJsonDocumentRemove(docId); + } else { + XmlStream stream = new XmlStream(); + stream.beginTag("remove"); + stream.addAttribute("documentid", docId); + stream.endTag(); + assert(stream.isFinalized()); + out.print(stream); + } + } + } catch (Exception e) { + System.err.println("Failed to output document: " + + e.getMessage()); + getControlHandler().abort(); + } + } + + private void writeJsonDocumentRemove(DocumentId docId) + throws IOException { + writeFeedStartOrRecordSeparator(); + out.write(JsonWriter.documentRemove(docId)); + } + + private void writeFeedStartOrRecordSeparator() { + if (first) { + out.println("["); + first = false; + } else { + out.println(","); + } + } + + private void writeFeedEnd() { + out.println("]"); + } + + public void onMapVisitorData(Map data) { + for (String key : data.keySet()) { + if (doStatistics) { + Integer i = statisticsMap.get(key); + if (i != null) { + statisticsMap.put(key, Integer.parseInt(data.get(key)) + i); + } else { + statisticsMap.put(key, Integer.parseInt(data.get(key))); + } + } else { + out.println(key + ":" + data.get(key)); + } + } + } + + public void onDocumentList(BucketId bucketId, List documents) { + out.println("Got document list of bucket " + bucketId.toString()); + for (DocumentListEntry entry : documents) { + entry.getDocument().setLastModified(entry.getTimestamp()); + onDocument(entry.getDocument(), entry.getTimestamp()); + } + } + + public void onEmptyBuckets(List bucketIds) { + StringBuilder buckets = new StringBuilder(); + for(BucketId bid : bucketIds) { + buckets.append(" "); + buckets.append(bid.toString()); + } + log.log(LogLevel.INFO, "Got EmptyBuckets: " + buckets); + } + + public synchronized void onDone() { + if (jsonOutput) { + writeFeedEnd(); + } + statisticsMap.dumpAll(); + super.onDone(); + } + } + + class ControlHandler extends VisitorControlHandler { + public void onProgress(ProgressToken token) { + if (showProgress) { + synchronized (printLock) { + if (lastLineIsProgress) { + System.err.print('\r'); + } + System.err.format("%.1f %% finished.", + token.percentFinished()); + lastLineIsProgress = true; + } + } + super.onProgress(token); + } + + public void onDone(CompletionCode code, String message) { + if (lastLineIsProgress) { + System.err.print('\n'); + lastLineIsProgress = false; + } + if (code != CompletionCode.SUCCESS) { + if (code == CompletionCode.ABORTED) { + System.err.println("Visitor aborted: " + message); + } else if (code == CompletionCode.TIMEOUT) { + System.err.println("Visitor timed out: " + message); + } else { + System.err.println("Visitor aborted due to unknown issue " + + code + ": " + message); + } + } else if (showProgress) { + System.err.println("Completed visiting."); + } + super.onDone(code, message); + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java new file mode 100644 index 00000000000..ff072b845de --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -0,0 +1,789 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespavisit; + +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.documentapi.VisitorControlHandler; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.log.LogSetup; +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.messagebus.StaticThrottlePolicy; +import com.yahoo.vespaclient.ClusterDef; +import com.yahoo.vespaclient.ClusterList; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.io.*; +import java.nio.charset.Charset; +import java.util.Map; + +/** + * Example client using visiting + * + * @author Einar M R Rosenvinge, based on work by Håkon Humberset + */ +public class VdsVisit { + private VdsVisitParameters params; + private MessageBusParams mbparams = new MessageBusParams(new LoadTypeSet()); + private VisitorSession session; + + private final VisitorSessionAccessorFactory sessionAccessorFactory; + private VisitorSessionAccessor sessionAccessor; + private ShutdownHookRegistrar shutdownHookRegistrar; + + public interface ShutdownHookRegistrar { + public void registerShutdownHook(Thread thread); + } + + public interface VisitorSessionAccessor { + public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException; + public void shutdown(); + } + + public interface VisitorSessionAccessorFactory { + public VisitorSessionAccessor createVisitorSessionAccessor(); + } + + private static class MessageBusVisitorSessionAccessor implements VisitorSessionAccessor { + private MessageBusDocumentAccess access; + + private MessageBusVisitorSessionAccessor(MessageBusParams mbparams) { + access = new MessageBusDocumentAccess(mbparams); + } + @Override + public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException { + return access.createVisitorSession(params); + } + + @Override + public void shutdown() { + access.shutdown(); + } + } + + private static class MessageBusVisitorSessionAccessorFactory implements VisitorSessionAccessorFactory { + MessageBusParams mbparams; + + private MessageBusVisitorSessionAccessorFactory(MessageBusParams mbparams) { + this.mbparams = mbparams; + } + + @Override + public VisitorSessionAccessor createVisitorSessionAccessor() { + return new MessageBusVisitorSessionAccessor(mbparams); + } + } + + private static class JvmRuntimeShutdownHookRegistrar implements ShutdownHookRegistrar { + @Override + public void registerShutdownHook(Thread thread) { + Runtime.getRuntime().addShutdownHook(thread); + } + } + + public VdsVisit() { + this.sessionAccessorFactory = new MessageBusVisitorSessionAccessorFactory(mbparams); + this.shutdownHookRegistrar = new JvmRuntimeShutdownHookRegistrar(); + } + + public VdsVisit(VisitorSessionAccessorFactory sessionAccessorFactory, + ShutdownHookRegistrar shutdownHookRegistrar) + { + this.sessionAccessorFactory = sessionAccessorFactory; + this.shutdownHookRegistrar = shutdownHookRegistrar; + } + + public static void main(String args[]) { + LogSetup.initVespaLogging("vespavisit"); + VdsVisit vdsVisit = new VdsVisit(); + + Options options = createOptions(); + + try { + ArgumentParser parser = new ArgumentParser(options); + vdsVisit.params = parser.parse(args); + if (vdsVisit.params == null) { + vdsVisit.printSyntax(options); + System.exit(0); + } + ClusterList clusterList = new ClusterList("client"); + vdsVisit.params.getVisitorParameters().setRoute( + resolveClusterRoute(clusterList, vdsVisit.params.getCluster())); + } catch (org.apache.commons.cli.ParseException e) { + System.err.println("Failed to parse arguments. Try --help for syntax. " + e.getMessage()); + System.exit(1); + } catch (IllegalArgumentException e) { + System.err.println(e.getMessage()); + System.exit(1); + } + + if (vdsVisit.params.isVerbose()) { + verbosePrintParameters(vdsVisit.params, System.err); + } + + try { + vdsVisit.run(); + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); + } + } + + private void printSyntax(Options options) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("vespavisit ", "Visit documents from VDS", options , ""); + } + + @SuppressWarnings("AccessStaticViaInstance") + protected static Options createOptions() { + Options options = new Options(); + options.addOption("h", "help", false, "Show this syntax page."); + + options.addOption(Option.builder("d") + .longOpt("datahandler") + .hasArg(true) + .argName("target") + .desc("Send results to the given target.") + .build()); + + options.addOption(Option.builder("s") + .longOpt("selection") + .hasArg(true) + .argName("selection") + .desc("What documents to visit.") + .build()); + + options.addOption(Option.builder("f") + .longOpt("from") + .hasArg(true) + .argName("timestamp") + .desc("Only visit from the given timestamp (microseconds).") + .type(Number.class) + .build()); + + options.addOption(Option.builder("t") + .longOpt("to") + .hasArg(true) + .argName("timestamp") + .desc("Only visit up to the given timestamp (microseconds).") + .type(Number.class).build()); + + options.addOption("e", "headersonly", false, "Only visit headers of documents.[Deprecated]"); + + options.addOption(Option.builder("l") + .longOpt("fieldset") + .hasArg(true) + .argName("fieldset") + .desc("Retrieve the specified fields only (see http://vespa.corp.yahoo.com/5/documentation/reference/fieldsets.html). Default is [all].") + .build()); + + options.addOption(Option.builder() + .longOpt("visitinconsistentbuckets") + .hasArg(false) + .desc("Don't wait for inconsistent buckets to become consistent.") + .build()); + + options.addOption(Option.builder("m") + .longOpt("maxpending") + .hasArg(true) + .argName("num") + .desc("Maximum pending messages to data handlers per storage visitor.") + .type(Number.class) + .build()); + + options.addOption(Option.builder() + .longOpt("maxpendingsuperbuckets") + .hasArg(true) + .argName("num") + .desc("Maximum pending visitor messages from the vespavisit client. If set, dynamic throttling of visitors will be disabled!") + .type(Number.class) + .build()); + + options.addOption(Option.builder("b") + .longOpt("maxbuckets") + .hasArg(true) + .argName("num") + .desc("Maximum buckets per visitor.") + .type(Number.class) + .build()); + + options.addOption("i", "printids", false, "Display only document identifiers."); + + options.addOption(Option.builder("p") + .longOpt("progress") + .hasArg(true) + .argName("file") + .desc("Use given file to track progress.") + .build()); + + options.addOption(Option.builder("o") + .longOpt("timeout") + .hasArg(true) + .argName("milliseconds") + .desc("Time out visitor after given time.") + .type(Number.class) + .build()); + + options.addOption(Option.builder("u") + .longOpt("buckettimeout") + .hasArg(true) + .argName("milliseconds") + .desc("Fail visitor if visiting a single bucket takes longer than this (default same as timeout)") + .type(Number.class) + .build()); + + options.addOption(Option.builder() + .longOpt("visitlibrary") + .hasArg(true) + .argName("string") + .desc("Use the given visitor library.") + .build()); + + options.addOption(Option.builder() + .longOpt("libraryparam") + .numberOfArgs(2) + .argName("key> 0) { + out.println("Adding the following library specific parameters:"); + for (Map.Entry entry : params.getLibraryParameters().entrySet()) { + out.println(" " + entry.getKey() + " = " + + new String(entry.getValue(), Charset.forName("utf-8"))); + } + } + if (params.getPriority() != DocumentProtocol.Priority.NORMAL_3) { + out.println("Visitor priority " + params.getPriority().name()); + } + if (params.skipBucketsOnFatalErrors()) { + out.println("Skip visiting super buckets with fatal errors."); + } + } + + private void onDocumentSelectionException(Exception e) { + System.err.println("Illegal document selection string '" + + params.getVisitorParameters().getDocumentSelection() + "'.\n"); + System.exit(1); + } + + private void onIllegalArgumentException(Exception e) { + System.err.println("Illegal arguments : \n"); + System.err.println(e.getMessage()); + System.exit(1); + } + + public void run() { + System.exit(doRun()); + } + + protected int doRun() { + VisitorParameters visitorParameters = params.getVisitorParameters(); + // If progress file already exists, create resume token from it + if (visitorParameters.getResumeFileName() != null && + !"".equals(visitorParameters.getResumeFileName())) + { + try { + File file = new File(visitorParameters.getResumeFileName()); + FileInputStream fos = new FileInputStream(file); + + StringBuilder builder = new StringBuilder(); + byte[] b = new byte[100000]; + int length; + + while ((length = fos.read(b)) > 0) { + builder.append(new String(b, 0, length)); + } + fos.close(); + visitorParameters.setResumeToken(new ProgressToken(builder.toString())); + + if (params.isVerbose()) { + System.err.format("Resuming visitor already %.1f %% finished.\n", + visitorParameters.getResumeToken().percentFinished()); + } + } catch (FileNotFoundException e) { + // Ignore; file has not been created yet but will be shortly. + } catch (IOException e) { + System.err.println("Could not open progress file: " + visitorParameters.getResumeFileName()); + e.printStackTrace(System.err); + return 1; + } + } + + initShutdownHook(); + sessionAccessor = sessionAccessorFactory.createVisitorSessionAccessor(); + + VdsVisitHandler handler; + + handler = new StdOutVisitorHandler( + params.isPrintIdsOnly(), + params.isVerbose(), + params.isVerbose(), + params.isVerbose(), + params.getStatisticsParts() != null, + params.getAbortOnClusterDown(), + params.getProcessTime(), + params.jsonOutput); + + if (visitorParameters.getResumeFileName() != null) { + handler.setProgressFileName(visitorParameters.getResumeFileName()); + } + + visitorParameters.setControlHandler(handler.getControlHandler()); + if (visitorParameters.getRemoteDataHandler() == null) { + visitorParameters.setLocalDataHandler(handler.getDataHandler()); + } + + if (params.getStatisticsParts() != null) { + String[] parts = params.getStatisticsParts().split(","); + for (String s : parts) { + visitorParameters.setLibraryParameter(s, "true"); + } + } + + try { + session = sessionAccessor.createVisitorSession(visitorParameters); + while (true) { + try { + if (session.waitUntilDone(params.getFullTimeout())) break; + } catch (InterruptedException e) {} + } + + if (visitorParameters.getTraceLevel() > 0) { + System.out.println(session.getTrace().toString()); + } + } catch (ParseException e) { + onDocumentSelectionException(e); + } catch (IllegalArgumentException e) { + onIllegalArgumentException(e); + } catch (Exception e) { + System.err.println("Document selection string was: " + visitorParameters.getDocumentSelection()); + System.err.println("Caught unexpected exception: "); + e.printStackTrace(System.err); + return 1; + } + if (visitorParameters.getControlHandler().getResult().code + == VisitorControlHandler.CompletionCode.SUCCESS) + { + return 0; + } else { + return 1; + } + } + + private void initShutdownHook() { + shutdownHookRegistrar.registerShutdownHook(new CleanUpThread()); + } + + class CleanUpThread extends Thread { + public void run() { + try { + if (session != null) { + session.destroy(); + } + } catch (IllegalStateException ise) { + //ignore this + } + try { + if (sessionAccessor != null) { + sessionAccessor.shutdown(); + } + } catch (IllegalStateException ise) { + //ignore this too + } + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java new file mode 100644 index 00000000000..ad3e0a0b359 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java @@ -0,0 +1,181 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespavisit; + +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.documentapi.VisitorControlHandler; +import com.yahoo.documentapi.VisitorDataHandler; +import com.yahoo.vdslib.VisitorStatistics; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +import java.util.Date; +import java.util.TimeZone; +import java.text.DateFormat; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; + +/** + * An abstract class that can be subclassed by different visitor handlers. + * + * @author Thomas Gundersen + */ +public abstract class VdsVisitHandler { + boolean showProgress; + boolean showStatistics; + boolean abortOnClusterDown; + boolean lastLineIsProgress = false; + String lastPercentage; + final Object printLock = new Object(); + + protected String progressFileName = ""; + + final VisitorControlHandler controlHandler = new ControlHandler(); + + public VdsVisitHandler(boolean showProgress, boolean showStatistics, boolean abortOnClusterDown) + { + this.showProgress = showProgress; + this.showStatistics = showStatistics; + this.abortOnClusterDown = abortOnClusterDown; + } + + public boolean getShowProgress() { + return showProgress; + } + + public boolean getShowStatistics() { + return showStatistics; + } + + public boolean getAbortOnClusterDown() { + return abortOnClusterDown; + } + + public boolean getLastLineIsProgress() { + return lastLineIsProgress; + } + + public void setLastLineIsProgress(boolean isProgress) { + lastLineIsProgress = isProgress; + } + + public String getLastPercentage() { + return lastPercentage; + } + + public void setLastPercentage(String lastPercentage) { + this.lastPercentage = lastPercentage; + } + + public Object getPrintLock() { + return printLock; + } + + public void onDone() { + } + + public String getProgressFileName() { + return progressFileName; + } + + public void setProgressFileName(String progressFileName) { + this.progressFileName = progressFileName; + } + + public VisitorControlHandler getControlHandler() { return controlHandler; } + public abstract VisitorDataHandler getDataHandler(); + + class ControlHandler extends VisitorControlHandler { + VisitorStatistics statistics; + + public void onProgress(ProgressToken token) { + if (progressFileName.length() > 0) { + try { + synchronized (token) { + File file = new File(progressFileName + ".tmp"); + FileOutputStream fos = new FileOutputStream(file); + fos.write(token.toString().getBytes()); + fos.close(); + file.renameTo(new File(progressFileName)); + } + } + catch (IOException e) { + e.printStackTrace(); + } + } + if (showProgress) { + synchronized (printLock) { + DecimalFormat df = new DecimalFormat("#.#"); + String percentage = df.format(token.percentFinished()); + if (!percentage.equals(lastPercentage)) { + if (lastLineIsProgress) { + System.err.print('\r'); + } + System.err.print(percentage + " % finished."); + lastLineIsProgress = true; + lastPercentage = percentage; + } + } + } + super.onProgress(token); + } + + @Override + public void onVisitorStatistics(VisitorStatistics visitorStatistics) { + statistics = visitorStatistics; + } + + private String getDateTime() { + DateFormat dateFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss zzz"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + Date date = new Date(); + return dateFormat.format(date); + } + + public void onVisitorError(String message) { + synchronized (printLock) { + if (lastLineIsProgress) { + System.err.print('\r'); + lastLineIsProgress = false; + } + System.err.println("Visitor error (" + getDateTime() + "): " + + message); + if (abortOnClusterDown && + !isDone() && + (message.lastIndexOf("Could not resolve")>=0 || + message.lastIndexOf("don't allow external load")>=0)) { + System.err.println("Aborting visitor as " + + "--abortonclusterdown flag is set."); + abort(); + } + } + } + public void onDone(CompletionCode code, String message) { + if (lastLineIsProgress) { + System.err.print('\n'); + lastLineIsProgress = false; + } + if (code != CompletionCode.SUCCESS) { + if (code == CompletionCode.ABORTED) { + System.err.println("Visitor aborted: " + message); + } else if (code == CompletionCode.TIMEOUT) { + System.err.println("Visitor timed out: " + message); + } else { + System.err.println("Visitor aborted due to unknown issue " + + code + ": " + message); + } + } else { + if (showProgress) { + System.err.println("Completed visiting."); + } + if (showStatistics) { + System.err.println("*** Visitor statistics"); + System.err.println(statistics == null ? "Nothing visited" : statistics.toString()); + } + } + super.onDone(code, message); + } + } +} diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java new file mode 100644 index 00000000000..3ef0619cfd8 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java @@ -0,0 +1,286 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespavisit; + +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.VisitorControlHandler; +import com.yahoo.documentapi.VisitorDataHandler; +import com.yahoo.documentapi.VisitorDestinationParameters; +import com.yahoo.documentapi.VisitorDestinationSession; +import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; +import com.yahoo.documentapi.messagebus.MessageBusParams; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.log.LogLevel; +import com.yahoo.log.LogSetup; +import com.yahoo.messagebus.network.Identity; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +import java.lang.reflect.Constructor; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.logging.Logger; + +/** + * Example client using visiting + * + * @author Einar M R Rosenvinge, based on work by Håkon Humberset + */ +public class VdsVisitTarget { + private static final Logger log = Logger.getLogger(VdsVisitTarget.class.getName()); + + private boolean printIds = false; + DocumentAccess access; + VisitorDestinationSession session; + String slobrokAddress = null; + int port = -1; + private boolean verbose = false; + private int processTime = 0; + private String handlerClassName = StdOutVisitorHandler.class.getName(); + private String[] handlerArgs = null; + + public boolean isPrintIds() { + return printIds; + } + + public String getSlobrokAddress() { + return slobrokAddress; + } + + public boolean isVerbose() { + return verbose; + } + + public int getPort() { + return port; + } + + public int getProcessTime() { + return processTime; + } + + public String getHandlerClassName() { + return handlerClassName; + } + + public String[] getHandlerArgs() { + return handlerArgs; + } + + public static void main(String args[]) { + LogSetup.initVespaLogging("vespavisittarget"); + VdsVisitTarget visitTarget = new VdsVisitTarget(); + + + try { + visitTarget.parseArguments(args); + visitTarget.initShutdownHook(); + visitTarget.run(); + System.exit(0); + } catch (HelpShownException e) { + System.exit(0); + } catch (IllegalArgumentException e) { + System.err.println(e.getMessage()); + System.exit(1); + } catch (org.apache.commons.cli.ParseException e) { + System.err.println("Failed to parse arguments. Try --help for syntax. " + e.getMessage()); + System.exit(1); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private static Options createOptions() { + Options options = new Options(); + + options.addOption("h", "help", false, "Show this syntax page."); + + options.addOption(Option.builder("s") + .longOpt("bindtoslobrok") + .hasArg(true) + .argName("address") + .desc("Bind to the given slobrok address.") + .build()); + + options.addOption(Option.builder("t") + .longOpt("bindtosocket") + .hasArg(true) + .argName("port") + .desc("Bind to the given TCP port") + .type(Number.class) + .build()); + + options.addOption(Option.builder("p") + .longOpt("processtime") + .hasArg(true) + .argName("msecs") + .desc("Sleep this amount of millisecs before processing message. (Debug option for pretending to be slow client).") + .type(Number.class) + .build()); + + options.addOption(Option.builder("c") + .longOpt("visithandler") + .hasArg(true) + .argName("classname") + .desc("Use the given class as a visit handler (defaults to StdOutVisitorHandler)") + .build()); + + options.addOption(Option.builder("o") + .longOpt("visitoptions") + .hasArg(true) + .argName("args") + .desc("Option arguments to pass through to the visitor handler instance") + .build()); + + options.addOption("i", "printids", false, "Display only document identifiers."); + options.addOption("v", "verbose", false, "Indent XML, show progress and info on STDERR."); + + return options; + } + + private void printSyntax(Options options) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("vespavisittarget ", "Retrieve results from a visitor", options , + "One, and only one, of the binding options must be present.\n" + + "\n" + + "For more detailed information, such as defaults and format of\n" + + "arguments, refer to 'man vespavisittarget'.\n"); + } + + class HelpShownException extends Exception {} + + void parseArguments(String args[]) throws ParseException, HelpShownException { + Options options = createOptions(); + + CommandLineParser parser = new DefaultParser(); + CommandLine line = parser.parse(options, args); + + if (line.hasOption("h")) { + printSyntax(options); + throw new HelpShownException(); + } + if (line.hasOption("s")) { + slobrokAddress = line.getOptionValue("s"); + } + if (line.hasOption("t")) { + port = ((Number) line.getParsedOptionValue("t")).intValue(); + } + if (line.hasOption("i")) { + printIds = true; + } + if (line.hasOption("p")) { + processTime = ((Number) line.getParsedOptionValue("p")).intValue(); + } + if (line.hasOption("v")) { + verbose = true; + } + if (line.hasOption("c")) { + handlerClassName = line.getOptionValue("c"); + } + if (line.hasOption("o")) { + handlerArgs = line.getOptionValue("o").split(" "); + } + + if (!(slobrokAddress == null ^ port == -1)) { + throw new IllegalArgumentException("You must specify one, and only one, binding option"); + } + if (port != -1 && port < 0 || port > 65535) { + throw new IllegalArgumentException("The port must be in the range 0-65535"); + } + if (verbose) { + if (port != -1) { + System.err.println("Binding to socket " + getTcpAddress()); + } else { + System.err.println("Binding to slobrok address: " + slobrokAddress + "/visit-destination"); + } + } + } + + private String getTcpAddress() { + try { + InetAddress addr = InetAddress.getLocalHost(); + String hostname = addr.getHostName(); + return "tcp/" + hostname + ":" + port + "/visit-destination"; + } catch (UnknownHostException e) { + System.err.println("Failed to detect hostname."); + System.exit(1); + } + return ""; + } + + @SuppressWarnings("unchecked") + public void run() throws Exception { + initShutdownHook(); + log.log(LogLevel.DEBUG, "Starting VdsVisitTarget"); + MessageBusParams mbusParams = new MessageBusParams(new LoadTypeSet()); + mbusParams.getRPCNetworkParams().setIdentity(new Identity(slobrokAddress)); + + if (port > 0) { + mbusParams.getRPCNetworkParams().setListenPort(port); + } + + access = new MessageBusDocumentAccess(mbusParams); + + VdsVisitHandler handler; + + Class cls = Thread.currentThread().getContextClassLoader() + .loadClass(handlerClassName); + try { + // Any custom data handlers may have a constructor that takes in args, + // so that the user can pass cmd line options to them + Class[] consTypes = new Class[] { boolean.class, boolean.class, + boolean.class, boolean.class, boolean.class, + boolean.class, int.class, String[].class }; + Constructor cons = cls.getConstructor(consTypes); + handler = (VdsVisitHandler)cons.newInstance( + printIds, verbose, verbose, verbose, false, false, + processTime, handlerArgs); + } catch (NoSuchMethodException e) { + // Retry, this time matching the StdOutVisitorHandler constructor + // arg list + Class[] consTypes = new Class[] { boolean.class, boolean.class, + boolean.class, boolean.class, boolean.class, + boolean.class, int.class, boolean.class }; + Constructor cons = cls.getConstructor(consTypes); + handler = (VdsVisitHandler)cons.newInstance( + printIds, verbose, verbose, verbose, false, false, processTime, false); + } + + VisitorDataHandler dataHandler = handler.getDataHandler(); + VisitorControlHandler controlHandler = handler.getControlHandler(); + + VisitorDestinationParameters params = new VisitorDestinationParameters( + "visit-destination", dataHandler); + session = access.createVisitorDestinationSession(params); + while (!controlHandler.isDone()) { + Thread.sleep(1000); + } + } + + private void initShutdownHook() { + Runtime.getRuntime().addShutdownHook(new CleanUpThread()); + } + + class CleanUpThread extends Thread { + public void run() { + try { + if (session != null) { + session.destroy(); + } + } catch (IllegalStateException ise) { + //ignore this + } + try { + if (access != null) { + access.shutdown(); + } + } catch (IllegalStateException ise) { + //ignore this too + } + } + } +} diff --git a/vespaclient-java/src/main/sh/vds-document-statistics.sh b/vespaclient-java/src/main/sh/vds-document-statistics.sh new file mode 100755 index 00000000000..3677137dbd4 --- /dev/null +++ b/vespaclient-java/src/main/sh/vds-document-statistics.sh @@ -0,0 +1,20 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +function help { + echo "Usage: vds-document-statistics [ category, ... ]" + echo " Where category is one or more of: user, group, scheme, namespace" + echo "" + echo "vds-document-statistics generates documents counts based on one or more categories." + exit 0 +} +if [ "$1" == "-h" ]; then + help +fi +if [ "$1" == "" ]; then + help +fi +export MALLOC_ARENA_MAX=1 #Does not need fast allocation +exec java -Xms32m -Xmx128m $(getJavaOptionsIPV46) -cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespavisit.Main --statistics "$1" diff --git a/vespaclient-java/src/main/sh/vdsstat.sh b/vespaclient-java/src/main/sh/vdsstat.sh new file mode 100644 index 00000000000..ef3e2cdbe20 --- /dev/null +++ b/vespaclient-java/src/main/sh/vdsstat.sh @@ -0,0 +1,13 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +export MALLOC_ARENA_MAX=1 #Does not need fast allocation +exec java \ +-server -enableassertions \ +-XX:ThreadStackSize=512 \ +-XX:MaxJavaStackTraceDepth=-1 \ +-Djava.awt.headless=true \ +-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespastat.Main "$@" diff --git a/vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh b/vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh new file mode 100755 index 00000000000..1a70fcb006d --- /dev/null +++ b/vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh @@ -0,0 +1,6 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +java $(getJavaOptionsIPV46) -cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.search.query.profile.DumpTool $@ diff --git a/vespaclient-java/src/main/sh/vespa-summary-benchmark.sh b/vespaclient-java/src/main/sh/vespa-summary-benchmark.sh new file mode 100755 index 00000000000..7534639d07b --- /dev/null +++ b/vespaclient-java/src/main/sh/vespa-summary-benchmark.sh @@ -0,0 +1,15 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +export VESPA_LOG_TARGET=file:/dev/null +export MALLOC_ARENA_MAX=1 # Does not need fast allocation +java \ +-server -enableassertions \ +-XX:ThreadStackSize=512 \ +-XX:MaxJavaStackTraceDepth=-1 \ +-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ +-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true \ +-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespasummarybenchmark.VespaSummaryBenchmark "$@" diff --git a/vespaclient-java/src/main/sh/vespadestination.sh b/vespaclient-java/src/main/sh/vespadestination.sh new file mode 100755 index 00000000000..f2168cb6db2 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespadestination.sh @@ -0,0 +1,12 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +export MALLOC_ARENA_MAX=1 #Does not need fast allocation +exec java \ +-server -enableassertions \ +-XX:ThreadStackSize=512 \ +-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ +-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar:$CLASSPATH com.yahoo.dummyreceiver.DummyReceiver "$@" diff --git a/vespaclient-java/src/main/sh/vespafeeder.sh b/vespaclient-java/src/main/sh/vespafeeder.sh new file mode 100755 index 00000000000..f74bc794ed7 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespafeeder.sh @@ -0,0 +1,15 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +export VESPA_LOG_TARGET=file:/dev/null +export MALLOC_ARENA_MAX=1 # Does not need fast allocation +java \ +-server -enableassertions \ +-XX:ThreadStackSize=512 \ +-XX:MaxJavaStackTraceDepth=-1 \ +-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ +-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true \ +-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespafeeder.VespaFeeder "$@" diff --git a/vespaclient-java/src/main/sh/vespaget.sh b/vespaclient-java/src/main/sh/vespaget.sh new file mode 100644 index 00000000000..514ff170742 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespaget.sh @@ -0,0 +1,14 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +export MALLOC_ARENA_MAX=1 #Does not need fast allocation +exec java \ +-server -enableassertions \ +-XX:ThreadStackSize=512 \ +-XX:MaxJavaStackTraceDepth=-1 \ +-Djava.awt.headless=true \ +-DVESPA_LOG_LEVEL="all -debug -spam -config -info -event" \ +-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespaget.Main "$@" diff --git a/vespaclient-java/src/main/sh/vespavisit.1 b/vespaclient-java/src/main/sh/vespavisit.1 new file mode 100644 index 00000000000..b9a6c488bf9 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespavisit.1 @@ -0,0 +1,159 @@ +.TH VESPAVISIT 1 2008-03-07 "Vespa" "Vespa Documentation" +.SH NAME +vespavisit \- Visit documents from a Vespa installation +.SH SYNPOSIS +.B vespavisit +[\fIOPTION\fR]... +.SH DESCRIPTION +.PP +In the regular case, retrieve documents stored in VESPA, and either print +them to STDOUT or send them to a given MessageBus route. +.PP +A Vespa visit operation processes a set of stored documents, in undefined +order, locally on the storage nodes where they are stored. A visitor library +available on all storage nodes will receive the documents stored locally, and +can process these and send messages to the visitor data handler. The regular +case is to use the DumpVisitor library to merely send the documents themselves +in blocks back to the data handler, which by default is this client that will +write the documents to STDOUT. +.PP +Mandatory arguments to long options are mandatory for short options too. +Short options can not currently be concatenated together. +.TP +\fB\-s\fR, \fB\-\-selection\fR \fISELECTION\fR +A document selection string, specifying what documents to visit. Documentation +on the language itself can be found in the documentation. Note that this argument +should probably be quoted to prevent your shell from invalidating your +selection. +.TP +\fB\-f\fR, \fB\-\-from\fR \fITIME\fR +If this option is given, only documents from given timestamp or newer will be +visited. The time is given in microseconds since 1970. +.TP +\fB\-t\fR, \fB\-\-to\fR \fITIME\fR +If this option is given, only documents up to and including the given timestamp +will be visited. The time is given in microseconds since 1970. +.TP +\fB\-e\fR, \fB\-\-headersonly\fR +By default, the whole documents stored are processed. If this option is given +only the header parts of documents will be processed. By defining the big +document fields as body fields, you can efficiently visit all the header fields +using this option. +.TP +\fB\-i\fR, \fB\-\-printids\fR +Using this option, only the document identifiers will be printed to STDOUT. +In addition, if visiting removes, an additional tag will be added so you can +see whether document has been removed or not. This option implies headers only +visiting, and can only be used if no datahandler is specified. +.TP +\fB\-d\fR, \fB\-\-datahandler\fR \fIVISITTARGET\fR +The data handler is the destination of messages sent from the visitor library. +By default, the data handler is the vespavisit process you start, which will +merely print all returned data to STDOUT. A visit target can be specified +instead. See the chapter below on visit targets. +.TP +\fB\-p\fR, \fB\-\-progress\fR \fIFILE\fR +By setting a progress file, current visitor progress will be saved to this +file at regular intervals. If this file exists on startup, the visitor will +continue from this point. +.TP +\fB\-o\fR, \fB\-\-timeout\fR \fITIMEOUT\fR +Time out the visitor after given number of milliseconds. +.TP +\fB\-r\fR, \fB\-\-visitremoves\fR +By default, only documents existing in Vespa will be processed. By giving +this option, also entries identifying documents previously existing will +be returned. This is useful for secondary copies of data that wants to know +whether documents it has stored has been removed. Note that documents deleted +a long time ago will no longer be tracked. Vespa keeps remove entries for +a configurable amount of time. +.TP +\fB\-m\fR, \fB\-\-maxpending\fR \fINUM\fR +Maximum pending docblock messages to data handlers. This may be used to +increase or reduce visiting speed, but should not be set too high so that data +handlers run out of memory. To get an estimate of memory consumption on each +data handler, multiply maxpending with defaultdocblocksize in stor-visitor +config and divide by number of data handlers. Default value for maxpending is +16. +.TP +\fB\-c\fR, \fB\-\-cluster\fR \fICLUSTER\fR +Visit the given VDS cluster. +.TP +\fB\-v\fR, \fB\-\-verbose\fR +More verbose output. Indent XML and add progress and info to STDERR. +.TP +\fB\-h\fR, \fB\-\-help\fR +Shows a short syntax reminder. +.PP +Advanced options: +.PP +The below options are used for advanced usage or for testing. +.TP +\fB\-\-visitlibrary\fR \fILIBRARY\fR +By default, the DumpVisitor library, sending documents back to the data handler, +is used when visiting. Another library can be specified using this option. The +library filename should be the name given here, with lib prepended and .so +appended. +.TP +\fB\-\-libraryparam\fR \fIKEY\fR \fIVALUE\fR +The default DumpVisitor library has no options to set, but custom libraries +may need user specifiable options. Here such options can be specified. Look +at visitor library documentation for legal parameters. +.TP +\fB\-\-polling\fR \fIarg\fR +The document API implements both a polling and a callback visitor API. The +callback API is most efficient and used by default. The polling API might be +simpler for users used to such APIs. Some VESPA system tests use this option +to test that the polling API works. +.TP +\fB\-\-visitinconsistentbuckets\fR +In some cases Vespa may temporarily be in an inconsistent state, that is, +different nodes contain different copies of the data. Collections of documents +are grouped into so-called buckets. The normal behavior of visiting is to wait +for the inconsistencies to resolve before actually visiting the data. This +might be a problem for time critical applications. Setting this option will +result in the bucket copy with most documents to be visited in case of +inconsistencies, which means that the data returned by the visitor are not +guaranteed to be correct. +.SH VISIT TARGET +Results from visiting can be sent to many different kind of targets. +.TP +\fBMessage bus routes\fR +You can specify a message bus route name directly, and this route will be used +to send the results. This is typically used when doing reprocessing within +Vespa. Message bus routes are set up in the application package. In addition +some routes may have been autogenerated in simple setups, for instance a +route called \fIdefault\fR is generated if your setup is so simple that Vespa +can guess where you want to send your data. +.TP +\fBSlobrok address\fR +You can also specify a slobrok address for data to be sent to. A slobrok address +is a slash separated path where you can use asterisk to mean any element within +this path. For instance, if you have a docproc cluster called \fImydpcluster\fR +it will have registered its nodes with slobrok names like +\fIdocproc/cluster.mydpcluster/docproc/0/feed_processor\fR, where the 0 here +indicates the first node in the cluster. You can thus specify to send visit data +to this docproc cluster by stating a slobrok address of +\fIdocproc/cluster.mydpcluster/docproc/*/feed_processor\fR. Note that this will +not send all the data to one or all the nodes. The data sent from the visitor +will be distributed among the matching nodes, but each message will just be sent +to one node. + +Slobrok names may also be used if you use the \fBvespavisittarget\fR tool to +retrieve the data at some location. If you start vespavisittarget on two nodes, +listening to slobrok names \fImynode/0/visit-destination\fR and +\fImynode/1/visit-destination\fR you can send the results to these nodes by +specifying \fImynode/*/visit-destination\fR as the data handler. See +\fBman vespavisittarget\fR for naming conventions used for such targets. +.TP +\fBTCP socket\fR +TCP sockets can also be specified directly. This requires that the endpoint +speaks FNET RPC though. This is typically done, either by using the +\fBvespavisittarget\fR tool, or by using a visitor destination programmatically +by using utility class in the document API. A socket address looks like the +following: tcp/\fIhostname\fR:\fIport\fR/\fIservicename\fR. For instance, an +address generated by the \fBvespavisittarget\fR tool might look like the +following: \fItcp/myhost.com:12345/visit-destination\fR. + +.SH AUTHOR +Written by Haakon Humberset. diff --git a/vespaclient-java/src/main/sh/vespavisit.sh b/vespaclient-java/src/main/sh/vespavisit.sh new file mode 100755 index 00000000000..eb6c9487b88 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespavisit.sh @@ -0,0 +1,14 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +export MALLOC_ARENA_MAX=1 #Does not need fast allocation +exec java \ +-server -enableassertions \ +-XX:ThreadStackSize=512 \ +-XX:MaxJavaStackTraceDepth=-1 \ +-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ +-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true \ +-Xms128m -Xmx1024m $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar com.yahoo.vespavisit.VdsVisit "$@" diff --git a/vespaclient-java/src/main/sh/vespavisittarget.1 b/vespaclient-java/src/main/sh/vespavisittarget.1 new file mode 100644 index 00000000000..7f02215d558 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespavisittarget.1 @@ -0,0 +1,40 @@ +.TH VESPAVISITTARGET 1 2008-03-07 "Vespa" "Vespa Documentation" +.SH NAME +vespavisittarget \- An endpoint for documents visited from a Vespa installation +.SH SYNPOSIS +.B vespavisittarget +[\fIOPTION\fR]... +.SH DESCRIPTION +.PP +When visiting data from Vespa, you might not want to send the data back to the +controlling process. By using separate visitor targets you can divide load +between multiple nodes and have the controlling process run at another location. +The document API has utility classes to set up end points for visitor data from +Vespa. This application is a small tool that uses these utilities and merely +writes the data retrieved to STDOUT in XML format. +.PP +Mandatory arguments to long options are mandatory for short options too. +Short options can not currently be concatenated together. +.TP +\fB\-s\fR, \fB\-\-bindtoslobrok\fR \fISLOBROKADDRESS\fR +Bind to the given slobrok address. Note that the value \fI/visit-destination\fR +will be appended to the given address. +.TP +\fB\-t\fR, \fB\-\-bindtosocket\fR \fIPORT\fR +Bind to the given TCP socket. This will make sure we listen to the given port. +No slobrok registration is done using this option, so you need to specify +TCP socket address in visitors to get data sent to this destination. +.TP +\fB\-i\fR, \fB\-\-printids\fR +Using this option, only the document identifiers will be printed to STDOUT. +In addition, if visiting removes, an additional tag will be added so you can +see whether document has been removed or not. This option implies headers only +visiting, and can only be used if no datahandler is specified. +.TP +\fB\-v\fR, \fB\-\-verbose\fR +More verbose output. Indent XML. +.TP +\fB\-h\fR, \fB\-\-help\fR +Shows a short syntax reminder. +.SH AUTHOR +Written by Haakon Humberset. diff --git a/vespaclient-java/src/main/sh/vespavisittarget.sh b/vespaclient-java/src/main/sh/vespavisittarget.sh new file mode 100755 index 00000000000..7eb9fe17e04 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespavisittarget.sh @@ -0,0 +1,13 @@ +#!/bin/sh +test -z "$VESPA_HOME" && VESPA_HOME=/home/y + +. $VESPA_HOME/libexec/vespa/common-env.sh + +export MALLOC_ARENA_MAX=1 #Does not need fast allocation +exec java \ +-server -enableassertions \ +-XX:ThreadStackSize=512 \ +-XX:MaxJavaStackTraceDepth=-1 \ +-Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ +-XX:MaxDirectMemorySize=32m -Djava.awt.headless=true $(getJavaOptionsIPV46) \ +-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar:$CLASSPATH com.yahoo.vespavisit.VdsVisitTarget "$@" -- cgit v1.2.3