From 416f596b150ec159717bfd2f9b2ef70e4d4cd3dd Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Sat, 14 Jan 2023 18:41:49 +0100 Subject: Support direct tensor rendering --- .../com/yahoo/vespa/feed/perf/SimpleFeeder.java | 3 +-- .../java/com/yahoo/vespaget/ClientParameters.java | 15 ++++++++++++--- .../com/yahoo/vespaget/CommandLineOptions.java | 3 +++ .../java/com/yahoo/vespaget/DocumentRetriever.java | 2 +- .../com/yahoo/vespavisit/StdOutVisitorHandler.java | 22 +++++++++++++--------- .../main/java/com/yahoo/vespavisit/VdsVisit.java | 20 +++++++++++++++++--- 6 files changed, 47 insertions(+), 18 deletions(-) (limited to 'vespaclient-java/src/main/java/com/yahoo') diff --git a/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java b/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java index c40e2c21561..7f85f37436b 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java @@ -308,6 +308,7 @@ public class SimpleFeeder implements ReplyHandler { return new DocumentUpdate(deserializer); } } + @Override public FeedOperation read() throws Exception { int read = readExact(in, prefix); @@ -352,8 +353,6 @@ public class SimpleFeeder implements ReplyHandler { return new JsonDestination(params.getDumpStream(), failure, numReplies); } - - @SuppressWarnings("deprecation") SimpleFeeder(FeederParams params) { inputStreams = params.getInputStreams(); out = params.getStdOut(); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java index 7e464431f9a..91837cb4b09 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java @@ -39,13 +39,15 @@ public class ClientParameters { public final boolean jsonOutput; // Output JSON tensors in short form public final boolean tensorShortForm; - + // Output JSON tensorvalues directly + public final boolean tensorDirectValues; private ClientParameters( boolean help, Iterator documentIds, boolean printIdsOnly, String fieldSet, String route, String cluster, String configId, boolean showDocSize, double timeout, boolean noRetry, int traceLevel, - DocumentProtocol.Priority priority, boolean jsonOutput, boolean tensorShortForm) { + DocumentProtocol.Priority priority, boolean jsonOutput, boolean tensorShortForm, + boolean tensorDirectValues) { this.help = help; this.documentIds = documentIds; @@ -61,6 +63,7 @@ public class ClientParameters { this.priority = priority; this.jsonOutput = jsonOutput; this.tensorShortForm = tensorShortForm; + this.tensorDirectValues = tensorDirectValues; } public static class Builder { @@ -78,6 +81,7 @@ public class ClientParameters { private DocumentProtocol.Priority priority; private boolean jsonOutput; private boolean tensorShortForm; + private boolean tensorDirectValues; public Builder setHelp(boolean help) { this.help = help; @@ -149,10 +153,15 @@ public class ClientParameters { return this; } + public Builder setTensorDirectValues(boolean tensorDirectValues) { + this.tensorDirectValues = tensorDirectValues; + return this; + } + public ClientParameters build() { return new ClientParameters( help, documentIds, printIdsOnly, fieldSet, route, cluster, configId, - showDocSize, timeout, noRetry, traceLevel, priority, jsonOutput, tensorShortForm); + showDocSize, timeout, noRetry, traceLevel, priority, jsonOutput, tensorShortForm, tensorDirectValues); } } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java index b059ca6e62a..f13ed13b92a 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java @@ -39,6 +39,7 @@ public class CommandLineOptions { public static final String JSONOUTPUT_OPTION = "jsonoutput"; public static final String XMLOUTPUT_OPTION = "xmloutput"; public static final String SHORTTENSORS_OPTION = "shorttensors"; + public static final String DIRECTTENSORS_OPTION = "directtensors"; private final Options options = createOptions(); private final InputStream stdIn; @@ -167,6 +168,7 @@ public class CommandLineOptions { boolean jsonOutput = cl.hasOption(JSONOUTPUT_OPTION); boolean xmlOutput = cl.hasOption(XMLOUTPUT_OPTION); boolean shortTensors = cl.hasOption(SHORTTENSORS_OPTION); + boolean directTensors = cl.hasOption(DIRECTTENSORS_OPTION); int trace = getTrace(cl); DocumentProtocol.Priority priority = getPriority(cl); double timeout = getTimeout(cl); @@ -218,6 +220,7 @@ public class CommandLineOptions { .setTimeout(timeout) .setJsonOutput(!xmlOutput) .setTensorShortForm(shortTensors) + .setTensorDirectValues(directTensors) .build(); } catch (ParseException pe) { throw new IllegalArgumentException(pe.getMessage()); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java index 1a2f3424b3c..0f17fa587e4 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java @@ -168,7 +168,7 @@ public class DocumentRetriever { System.out.println(document.getId()); } else { if (params.jsonOutput) { - System.out.print(Utf8.toString(JsonWriter.toByteArray(document, params.tensorShortForm))); + System.out.print(Utf8.toString(JsonWriter.toByteArray(document, params.tensorShortForm, params.tensorDirectValues))); } else { System.out.print(document.toXML(" ")); } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java index 2ac0510a2a3..0c7ad81f212 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java @@ -33,30 +33,33 @@ import java.util.logging.Logger; * @author Thomas Gundersen */ public class StdOutVisitorHandler extends VdsVisitHandler { + private static final Logger log = Logger.getLogger( StdOutVisitorHandler.class.getName()); - private boolean printIds; - private boolean indentXml; - private int processTimeMilliSecs; - private PrintStream out; + private final boolean printIds; + private final boolean indentXml; + private final int processTimeMilliSecs; + private final PrintStream out; private final boolean jsonOutput; private final boolean tensorShortForm; + private final boolean tensorDirectValues; - private VisitorDataHandler dataHandler; + private final VisitorDataHandler dataHandler; public StdOutVisitorHandler(boolean printIds, boolean indentXml, boolean showProgress, boolean showStatistics, boolean doStatistics, boolean abortOnClusterDown, int processtime, boolean jsonOutput, - boolean tensorShortForm) + boolean tensorShortForm, + boolean tensorDirectValues) { this(printIds, indentXml, showProgress, showStatistics, doStatistics, abortOnClusterDown, processtime, - jsonOutput, tensorShortForm, createStdOutPrintStream()); + jsonOutput, tensorShortForm, tensorDirectValues, createStdOutPrintStream()); } StdOutVisitorHandler(boolean printIds, boolean indentXml, boolean showProgress, boolean showStatistics, boolean doStatistics, boolean abortOnClusterDown, int processtime, boolean jsonOutput, - boolean tensorShortForm, PrintStream out) + boolean tensorShortForm, boolean tensorDirectValues, PrintStream out) { super(showProgress, showStatistics, abortOnClusterDown); this.printIds = printIds; @@ -64,6 +67,7 @@ public class StdOutVisitorHandler extends VdsVisitHandler { this.processTimeMilliSecs = processtime; this.jsonOutput = jsonOutput; this.tensorShortForm = tensorShortForm; + this.tensorDirectValues = tensorDirectValues; this.out = out; this.dataHandler = new DataHandler(doStatistics); } @@ -174,7 +178,7 @@ public class StdOutVisitorHandler extends VdsVisitHandler { private void writeJsonDocument(Document doc) throws IOException { writeFeedStartOrRecordSeparator(); - out.write(JsonWriter.toByteArray(doc, tensorShortForm)); + out.write(JsonWriter.toByteArray(doc, tensorShortForm, tensorDirectValues)); } @Override diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java index f84cb7270bd..340d4a7eb81 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -371,6 +371,7 @@ public class VdsVisit { private int fullTimeout = 7 * 24 * 60 * 60 * 1000; private boolean jsonOutput = false; private boolean tensorShortForm = false; // TODO Vespa 9: change default to true + private boolean tensorDirectValues = false; // TODO Vespa 9: change default to true public VisitorParameters getVisitorParameters() { return visitorParameters; @@ -447,16 +448,25 @@ public class VdsVisit { public void setTensorShortForm(boolean tensorShortForm) { this.tensorShortForm = tensorShortForm; } + + public boolean tensorDirectValues() { + return tensorDirectValues; + } + + public void setTensorDirectValues(boolean tensorDirectValues) { + this.tensorDirectValues = tensorDirectValues; + } + } protected static class ArgumentParser { - private Options options; + private final Options options; public ArgumentParser(Options options) { this.options = options; } - public VdsVisitParameters parse(String args[]) throws org.apache.commons.cli.ParseException { + public VdsVisitParameters parse(String[] args) throws org.apache.commons.cli.ParseException { VdsVisitParameters allParams = new VdsVisitParameters(); VisitorParameters params = new VisitorParameters(""); CommandLineParser parser = new DefaultParser(); @@ -572,6 +582,9 @@ public class VdsVisit { if (line.hasOption("shorttensors")) { allParams.setTensorShortForm(true); } + if (line.hasOption("tensorvalues")) { + allParams.setTensorDirectValues(true); + } boolean jsonOutput = line.hasOption("jsonoutput"); boolean xmlOutput = line.hasOption("xmloutput"); @@ -743,7 +756,8 @@ public class VdsVisit { params.getAbortOnClusterDown(), params.getProcessTime(), params.jsonOutput, - params.tensorShortForm); + params.tensorShortForm, + params.tensorDirectValues); if (visitorParameters.getResumeFileName() != null) { handler.setProgressFileName(visitorParameters.getResumeFileName()); -- cgit v1.2.3