Add JSONL output support to `vespa-visit` CLI tool

JSONL output is enabled via new `--jsonl` argument. Mutually
exclusive with `--jsonoutput` and (deprecated) `--xmloutput`.

1 files changed, 68 insertions, 12 deletions

diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java
index c1bbe8711a5..aa708b1fde9 100644
--- a/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java
+++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java
@@ -1,15 +1,19 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespavisit;
 
+import com.yahoo.document.DataType;
 import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
 import com.yahoo.document.DocumentPut;
 import com.yahoo.document.DocumentType;
 import com.yahoo.document.TensorDataType;
+import com.yahoo.document.datatypes.StringFieldValue;
 import com.yahoo.document.datatypes.TensorFieldValue;
 import com.yahoo.documentapi.AckToken;
 import com.yahoo.documentapi.VisitorControlSession;
 import com.yahoo.documentapi.VisitorDataHandler;
 import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
+import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
 import org.junit.jupiter.api.Test;
@@ -26,23 +30,27 @@ import static org.mockito.Mockito.mock;
  * @author bjorncs
  */
 public class StdOutVisitorHandlerTest {
-    private boolean jsonOutput;
-
-    public void initStdOutVisitorHandlerTest(boolean jsonOutput) {
-        this.jsonOutput = jsonOutput;
-    }
 
     public static Object[] data() {
         return new Object[]{true, false};
     }
 
+    private static StdOutVisitorHandler.Params createHandlerParams(boolean jsonOutput, boolean tensorShortForm, boolean tensorDirectValues) {
+        var params = new StdOutVisitorHandler.Params();
+        params.outputFormat = jsonOutput ? StdOutVisitorHandler.OutputFormat.JSON
+                                         : StdOutVisitorHandler.OutputFormat.XML;
+        params.tensorShortForm = tensorShortForm;
+        params.tensorDirectValues = tensorDirectValues;
+        return params;
+    }
+
     @MethodSource("data")
     @ParameterizedTest(name = "jsonOutput={0}")
     void printing_ids_for_zero_documents_produces_empty_output(boolean jsonOutput) {
-        initStdOutVisitorHandlerTest(jsonOutput);
         ByteArrayOutputStream out = new ByteArrayOutputStream();
-        StdOutVisitorHandler visitorHandler =
-                new StdOutVisitorHandler(/*printIds*/true, false, false, false, false, false, 0, jsonOutput, false, false, new PrintStream(out, true));
+        var params = createHandlerParams(jsonOutput, false, false);
+        params.printIds = true;
+        StdOutVisitorHandler visitorHandler = new StdOutVisitorHandler(params, new PrintStream(out, true));
         VisitorDataHandler dataHandler = visitorHandler.getDataHandler();
         dataHandler.onDone();
         String output = out.toString();
@@ -52,10 +60,9 @@ public class StdOutVisitorHandlerTest {
     @MethodSource("data")
     @ParameterizedTest(name = "jsonOutput={0}")
     void printing_zero_documents_produces_empty_output(boolean jsonOutput) {
-        initStdOutVisitorHandlerTest(jsonOutput);
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         StdOutVisitorHandler visitorHandler =
-                new StdOutVisitorHandler(/*printIds*/false, false, false, false, false, false, 0, jsonOutput, false, false, new PrintStream(out, true));
+                new StdOutVisitorHandler(createHandlerParams(jsonOutput, false, false), new PrintStream(out, true));
         VisitorDataHandler dataHandler = visitorHandler.getDataHandler();
         dataHandler.onDone();
         String expectedOutput = jsonOutput ? "[]" : "";
@@ -71,8 +78,7 @@ public class StdOutVisitorHandlerTest {
         var putMsg = new PutDocumentMessage(new DocumentPut(doc));
 
         var out = new ByteArrayOutputStream();
-        var visitorHandler = new StdOutVisitorHandler(/*printIds*/false, false, false, false, false, false,
-                                                      0, true, tensorShortForm, tensorDirectValues, new PrintStream(out, true));
+        var visitorHandler = new StdOutVisitorHandler(createHandlerParams(true, tensorShortForm, tensorDirectValues), new PrintStream(out, true));
         var dataHandler = visitorHandler.getDataHandler();
         var controlSession = mock(VisitorControlSession.class);
         var ackToken = mock(AckToken.class);
@@ -100,4 +106,54 @@ public class StdOutVisitorHandlerTest {
         do_test_json_tensor_fields_rendering(true, false, expectedOutput);
     }
 
+    private static PutDocumentMessage createPutWithDocAndValue(DocumentType docType, String docId, String fieldValue) {
+        var doc = new Document(docType, docId);
+        doc.setFieldValue("bar", new StringFieldValue(fieldValue));
+        return new PutDocumentMessage(new DocumentPut(doc));
+    }
+
+    private static RemoveDocumentMessage createRemoveForDoc(String docId) {
+        return new RemoveDocumentMessage(new DocumentId(docId));
+    }
+
+    @MethodSource("data")
+    @ParameterizedTest(name = "jsonLinesFormat={0}")
+    void json_can_be_output_in_json_lines_format(boolean jsonLinesFormat) {
+        var docType = new DocumentType("foo");
+        docType.addField("bar", DataType.STRING);
+
+        var params = createHandlerParams(true, true, true);
+        params.outputFormat = jsonLinesFormat ? StdOutVisitorHandler.OutputFormat.JSONL
+                                              : StdOutVisitorHandler.OutputFormat.JSON;
+
+        var out            = new ByteArrayOutputStream();
+        var visitorHandler = new StdOutVisitorHandler(params, new PrintStream(out, true));
+        var dataHandler    = visitorHandler.getDataHandler();
+        var controlSession = mock(VisitorControlSession.class);
+        dataHandler.setSession(controlSession);
+
+        dataHandler.onMessage(createPutWithDocAndValue(docType, "id:baz:foo::1", "fluffy\nbunnies"),  mock(AckToken.class));
+        dataHandler.onMessage(createRemoveForDoc("id:baz:foo::2"),                                    mock(AckToken.class));
+        dataHandler.onMessage(createPutWithDocAndValue(docType, "id:baz:foo::3", "\r\ncool fox\r\n"), mock(AckToken.class));
+        dataHandler.onDone();
+
+        String output = out.toString().trim();
+        if (jsonLinesFormat) {
+            // JSONL; no implicit start/end array chars or trailing commas after objects
+            var expected = """
+                    {"id":"id:baz:foo::1","fields":{"bar":"fluffy\\nbunnies"}}
+                    {"remove":"id:baz:foo::2"}
+                    {"id":"id:baz:foo::3","fields":{"bar":"\\r\\ncool fox\\r\\n"}}""";
+            assertEquals(expected, output);
+        } else {
+            // non-JSONL; usual array of comma-separated objects form
+            var expected = """
+                    [
+                    {"id":"id:baz:foo::1","fields":{"bar":"fluffy\\nbunnies"}},
+                    {"remove":"id:baz:foo::2"},
+                    {"id":"id:baz:foo::3","fields":{"bar":"\\r\\ncool fox\\r\\n"}}]""";
+            assertEquals(expected, output);
+        }
+    }
+
 }