Support direct tensor rendering

7 files changed, 54 insertions, 25 deletions

diff --git a/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java b/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java
index c40e2c21561..7f85f37436b 100644
--- a/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java
+++ b/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java
@@ -308,6 +308,7 @@ public class SimpleFeeder implements ReplyHandler {
                 return new DocumentUpdate(deserializer);
             }
         }
+
         @Override
         public FeedOperation read() throws Exception {
             int read = readExact(in, prefix);
@@ -352,8 +353,6 @@ public class SimpleFeeder implements ReplyHandler {
         return new JsonDestination(params.getDumpStream(), failure, numReplies);
     }
 
-
-    @SuppressWarnings("deprecation")
     SimpleFeeder(FeederParams params) {
         inputStreams = params.getInputStreams();
         out = params.getStdOut();
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java
index 7e464431f9a..91837cb4b09 100644
--- a/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java
@@ -39,13 +39,15 @@ public class ClientParameters {
     public final boolean jsonOutput;
     // Output JSON tensors in short form
     public final boolean tensorShortForm;
-
+    // Output JSON tensorvalues directly
+    public final boolean tensorDirectValues;
 
     private ClientParameters(
             boolean help, Iterator<String> documentIds, boolean printIdsOnly,
             String fieldSet, String route, String cluster, String configId,
             boolean showDocSize, double timeout, boolean noRetry, int traceLevel,
-            DocumentProtocol.Priority priority, boolean jsonOutput, boolean tensorShortForm) {
+            DocumentProtocol.Priority priority, boolean jsonOutput, boolean tensorShortForm,
+            boolean tensorDirectValues) {
 
         this.help = help;
         this.documentIds = documentIds;
@@ -61,6 +63,7 @@ public class ClientParameters {
         this.priority = priority;
         this.jsonOutput = jsonOutput;
         this.tensorShortForm = tensorShortForm;
+        this.tensorDirectValues = tensorDirectValues;
     }
 
     public static class Builder {
@@ -78,6 +81,7 @@ public class ClientParameters {
         private DocumentProtocol.Priority priority;
         private boolean jsonOutput;
         private boolean tensorShortForm;
+        private boolean tensorDirectValues;
 
         public Builder setHelp(boolean help) {
             this.help = help;
@@ -149,10 +153,15 @@ public class ClientParameters {
             return this;
         }
 
+        public Builder setTensorDirectValues(boolean tensorDirectValues) {
+            this.tensorDirectValues = tensorDirectValues;
+            return this;
+        }
+
         public ClientParameters build() {
             return new ClientParameters(
                     help, documentIds, printIdsOnly, fieldSet, route, cluster, configId,
-                    showDocSize, timeout, noRetry, traceLevel, priority, jsonOutput, tensorShortForm);
+                    showDocSize, timeout, noRetry, traceLevel, priority, jsonOutput, tensorShortForm, tensorDirectValues);
         }
     }
 
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java
index b059ca6e62a..f13ed13b92a 100644
--- a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java
@@ -39,6 +39,7 @@ public class CommandLineOptions {
     public static final String JSONOUTPUT_OPTION = "jsonoutput";
     public static final String XMLOUTPUT_OPTION = "xmloutput";
     public static final String SHORTTENSORS_OPTION = "shorttensors";
+    public static final String DIRECTTENSORS_OPTION = "directtensors";
 
     private final Options options = createOptions();
     private final InputStream stdIn;
@@ -167,6 +168,7 @@ public class CommandLineOptions {
             boolean jsonOutput = cl.hasOption(JSONOUTPUT_OPTION);
             boolean xmlOutput = cl.hasOption(XMLOUTPUT_OPTION);
             boolean shortTensors = cl.hasOption(SHORTTENSORS_OPTION);
+            boolean directTensors = cl.hasOption(DIRECTTENSORS_OPTION);
             int trace = getTrace(cl);
             DocumentProtocol.Priority priority = getPriority(cl);
             double timeout = getTimeout(cl);
@@ -218,6 +220,7 @@ public class CommandLineOptions {
                     .setTimeout(timeout)
                     .setJsonOutput(!xmlOutput)
                     .setTensorShortForm(shortTensors)
+                    .setTensorDirectValues(directTensors)
                     .build();
         } catch (ParseException pe) {
             throw new IllegalArgumentException(pe.getMessage());
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java
index 1a2f3424b3c..0f17fa587e4 100644
--- a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java
+++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java
@@ -168,7 +168,7 @@ public class DocumentRetriever {
             System.out.println(document.getId());
         } else {
             if (params.jsonOutput) {
-                System.out.print(Utf8.toString(JsonWriter.toByteArray(document, params.tensorShortForm)));
+                System.out.print(Utf8.toString(JsonWriter.toByteArray(document, params.tensorShortForm, params.tensorDirectValues)));
             } else {
                 System.out.print(document.toXML("  "));
             }
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java
index 2ac0510a2a3..0c7ad81f212 100644
--- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java
+++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java
@@ -33,30 +33,33 @@ import java.util.logging.Logger;
  * @author Thomas Gundersen
  */
 public class StdOutVisitorHandler extends VdsVisitHandler {
+
     private static final Logger log = Logger.getLogger(
                                         StdOutVisitorHandler.class.getName());
-    private boolean printIds;
-    private boolean indentXml;
-    private int processTimeMilliSecs;
-    private PrintStream out;
+    private final boolean printIds;
+    private final boolean indentXml;
+    private final int processTimeMilliSecs;
+    private final PrintStream out;
     private final boolean jsonOutput;
     private final boolean tensorShortForm;
+    private final boolean tensorDirectValues;
 
-    private VisitorDataHandler dataHandler;
+    private final VisitorDataHandler dataHandler;
 
     public StdOutVisitorHandler(boolean printIds, boolean indentXml,
                                 boolean showProgress, boolean showStatistics, boolean doStatistics,
                                 boolean abortOnClusterDown, int processtime, boolean jsonOutput,
-                                boolean tensorShortForm)
+                                boolean tensorShortForm,
+                                boolean tensorDirectValues)
     {
         this(printIds, indentXml, showProgress, showStatistics, doStatistics, abortOnClusterDown, processtime,
-             jsonOutput, tensorShortForm, createStdOutPrintStream());
+             jsonOutput, tensorShortForm, tensorDirectValues, createStdOutPrintStream());
     }
 
     StdOutVisitorHandler(boolean printIds, boolean indentXml,
                          boolean showProgress, boolean showStatistics, boolean doStatistics,
                          boolean abortOnClusterDown, int processtime, boolean jsonOutput,
-                         boolean tensorShortForm, PrintStream out)
+                         boolean tensorShortForm, boolean tensorDirectValues, PrintStream out)
     {
         super(showProgress, showStatistics, abortOnClusterDown);
         this.printIds = printIds;
@@ -64,6 +67,7 @@ public class StdOutVisitorHandler extends VdsVisitHandler {
         this.processTimeMilliSecs = processtime;
         this.jsonOutput = jsonOutput;
         this.tensorShortForm = tensorShortForm;
+        this.tensorDirectValues = tensorDirectValues;
         this.out = out;
         this.dataHandler = new DataHandler(doStatistics);
     }
@@ -174,7 +178,7 @@ public class StdOutVisitorHandler extends VdsVisitHandler {
 
         private void writeJsonDocument(Document doc) throws IOException {
             writeFeedStartOrRecordSeparator();
-            out.write(JsonWriter.toByteArray(doc, tensorShortForm));
+            out.write(JsonWriter.toByteArray(doc, tensorShortForm, tensorDirectValues));
         }
 
         @Override
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java
index f84cb7270bd..340d4a7eb81 100644
--- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java
+++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java
@@ -371,6 +371,7 @@ public class VdsVisit {
         private int fullTimeout = 7 * 24 * 60 * 60 * 1000;
         private boolean jsonOutput = false;
         private boolean tensorShortForm = false; // TODO Vespa 9: change default to true
+        private boolean tensorDirectValues = false; // TODO Vespa 9: change default to true
 
         public VisitorParameters getVisitorParameters() {
             return visitorParameters;
@@ -447,16 +448,25 @@ public class VdsVisit {
         public void setTensorShortForm(boolean tensorShortForm) {
             this.tensorShortForm = tensorShortForm;
         }
+
+        public boolean tensorDirectValues() {
+            return tensorDirectValues;
+        }
+
+        public void setTensorDirectValues(boolean tensorDirectValues) {
+            this.tensorDirectValues = tensorDirectValues;
+        }
+
     }
 
     protected static class ArgumentParser {
-        private Options options;
+        private final Options options;
 
         public ArgumentParser(Options options) {
             this.options = options;
         }
 
-        public VdsVisitParameters parse(String args[]) throws org.apache.commons.cli.ParseException {
+        public VdsVisitParameters parse(String[] args) throws org.apache.commons.cli.ParseException {
             VdsVisitParameters allParams = new VdsVisitParameters();
             VisitorParameters params = new VisitorParameters("");
             CommandLineParser parser = new DefaultParser();
@@ -572,6 +582,9 @@ public class VdsVisit {
             if (line.hasOption("shorttensors")) {
                 allParams.setTensorShortForm(true);
             }
+            if (line.hasOption("tensorvalues")) {
+                allParams.setTensorDirectValues(true);
+            }
 
             boolean jsonOutput = line.hasOption("jsonoutput");
             boolean xmlOutput = line.hasOption("xmloutput");
@@ -743,7 +756,8 @@ public class VdsVisit {
                 params.getAbortOnClusterDown(),
                 params.getProcessTime(),
                 params.jsonOutput,
-                params.tensorShortForm);
+                params.tensorShortForm,
+                params.tensorDirectValues);
 
         if (visitorParameters.getResumeFileName() != null) {
             handler.setProgressFileName(visitorParameters.getResumeFileName());
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java
index a2e9f91d503..c1bbe8711a5 100644
--- a/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java
+++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java
@@ -42,7 +42,7 @@ public class StdOutVisitorHandlerTest {
         initStdOutVisitorHandlerTest(jsonOutput);
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         StdOutVisitorHandler visitorHandler =
-                new StdOutVisitorHandler(/*printIds*/true, false, false, false, false, false, 0, jsonOutput, false, new PrintStream(out, true));
+                new StdOutVisitorHandler(/*printIds*/true, false, false, false, false, false, 0, jsonOutput, false, false, new PrintStream(out, true));
         VisitorDataHandler dataHandler = visitorHandler.getDataHandler();
         dataHandler.onDone();
         String output = out.toString();
@@ -55,7 +55,7 @@ public class StdOutVisitorHandlerTest {
         initStdOutVisitorHandlerTest(jsonOutput);
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         StdOutVisitorHandler visitorHandler =
-                new StdOutVisitorHandler(/*printIds*/false, false, false, false, false, false, 0, jsonOutput, false, new PrintStream(out, true));
+                new StdOutVisitorHandler(/*printIds*/false, false, false, false, false, false, 0, jsonOutput, false, false, new PrintStream(out, true));
         VisitorDataHandler dataHandler = visitorHandler.getDataHandler();
         dataHandler.onDone();
         String expectedOutput = jsonOutput ? "[]" : "";
@@ -63,7 +63,7 @@ public class StdOutVisitorHandlerTest {
         assertEquals(expectedOutput, output);
     }
 
-    void do_test_json_tensor_fields_can_be_output_in_short_or_long_form(boolean tensorShortForm, String expectedOutput) {
+    void do_test_json_tensor_fields_rendering(boolean tensorShortForm, boolean tensorDirectValues, String expectedOutput) {
         var docType = new DocumentType("foo");
         docType.addField("bar", TensorDataType.getTensor(TensorType.fromSpec("tensor(x[3])")));
         var doc = new Document(docType, "id:baz:foo::tensor-stuff");
@@ -72,7 +72,7 @@ public class StdOutVisitorHandlerTest {
 
         var out = new ByteArrayOutputStream();
         var visitorHandler = new StdOutVisitorHandler(/*printIds*/false, false, false, false, false, false,
-                                                      0, true, tensorShortForm, new PrintStream(out, true));
+                                                      0, true, tensorShortForm, tensorDirectValues, new PrintStream(out, true));
         var dataHandler = visitorHandler.getDataHandler();
         var controlSession = mock(VisitorControlSession.class);
         var ackToken = mock(AckToken.class);
@@ -88,8 +88,8 @@ public class StdOutVisitorHandlerTest {
     void json_tensor_fields_can_be_output_in_long_form() {
         var expectedOutput = """
         [
-        {"id":"id:baz:foo::tensor-stuff","fields":{"bar":{"cells":[{"address":{"x":"0"},"value":1.0},{"address":{"x":"1"},"value":2.0},{"address":{"x":"2"},"value":3.0}]}}}]""";
-        do_test_json_tensor_fields_can_be_output_in_short_or_long_form(false, expectedOutput);
+        {"id":"id:baz:foo::tensor-stuff","fields":{"bar":{"type":"tensor(x[3])","cells":[{"address":{"x":"0"},"value":1.0},{"address":{"x":"1"},"value":2.0},{"address":{"x":"2"},"value":3.0}]}}}]""";
+        do_test_json_tensor_fields_rendering(false, false, expectedOutput);
     }
 
     @Test
@@ -97,7 +97,7 @@ public class StdOutVisitorHandlerTest {
         var expectedOutput = """
         [
         {"id":"id:baz:foo::tensor-stuff","fields":{"bar":{"type":"tensor(x[3])","values":[1.0,2.0,3.0]}}}]""";
-        do_test_json_tensor_fields_can_be_output_in_short_or_long_form(true, expectedOutput);
+        do_test_json_tensor_fields_rendering(true, false, expectedOutput);
     }
 
 }