Revert "Merge pull request #30067 from vespa-engine/revert-30038-jonmv/leaner-token-buffer"

This reverts commit b771fbe5fe648cf4c64e04341542e11c2e21cb9d, reversing
changes made to 7b578506b5c4c59f4273e74af1f0db4a74f82175.

9 files changed, 666 insertions, 327 deletions

diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java
index 3e1743b8d45..08d1fe688ed 100644
--- a/document/src/main/java/com/yahoo/document/json/JsonReader.java
+++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java
@@ -18,6 +18,7 @@ import java.io.InputStream;
 import java.util.Optional;
 
 import static com.yahoo.document.json.JsonReader.ReaderState.END_OF_FEED;
+import static com.yahoo.document.json.document.DocumentParser.FIELDS;
 import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart;
 
 /**
@@ -60,7 +61,7 @@ public class JsonReader {
      * @param docIdString document ID
      * @return the parsed document operation
      */
-    public ParsedDocumentOperation readSingleDocument(DocumentOperationType operationType, String docIdString) {
+    ParsedDocumentOperation readSingleDocument(DocumentOperationType operationType, String docIdString) {
         DocumentId docId = new DocumentId(docIdString);
         DocumentParseInfo documentParseInfo;
         try {
@@ -78,6 +79,54 @@ public class JsonReader {
         return operation;
     }
 
+    /**
+     * Reads a JSON which is expected to contain only the "fields" object of a document,
+     * and where other parameters, like the document ID and operation type, are supplied by other means.
+     *
+     * @param operationType the type of operation (update or put)
+     * @param docIdString document ID
+     * @return the parsed document operation
+     */
+    public ParsedDocumentOperation readSingleDocumentStreaming(DocumentOperationType operationType, String docIdString) {
+        try {
+            DocumentId docId = new DocumentId(docIdString);
+            DocumentParseInfo documentParseInfo = new DocumentParseInfo();
+            documentParseInfo.documentId = docId;
+            documentParseInfo.operationType = operationType;
+
+            if (JsonToken.START_OBJECT != parser.nextValue())
+                throw new IllegalArgumentException("expected start of root object, got " + parser.currentToken());
+
+            parser.nextValue();
+            if ( ! FIELDS.equals(parser.getCurrentName()))
+                throw new IllegalArgumentException("expected field \"fields\", but got " + parser.getCurrentName());
+
+            if (JsonToken.START_OBJECT != parser.currentToken())
+                throw new IllegalArgumentException("expected start of \"fields\" object, got " + parser.currentToken());
+
+            documentParseInfo.fieldsBuffer = new LazyTokenBuffer(parser);
+            VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(typeManager.getIgnoreUndefinedFields());
+            ParsedDocumentOperation operation = vespaJsonDocumentReader.createDocumentOperation(
+                    getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager), documentParseInfo);
+
+            if ( ! documentParseInfo.fieldsBuffer.isEmpty())
+                throw new IllegalArgumentException("expected all content to be consumed by document parsing, but " +
+                                                   documentParseInfo.fieldsBuffer.nesting() + " levels remain");
+
+            if (JsonToken.END_OBJECT != parser.currentToken())
+                throw new IllegalArgumentException("expected end of \"fields\" object, got " + parser.currentToken());
+            if (JsonToken.END_OBJECT != parser.nextToken())
+                throw new IllegalArgumentException("expected end of root object, got " + parser.currentToken());
+            if (null != parser.nextToken())
+                throw new IllegalArgumentException("expected end of input, got " + parser.currentToken());
+
+            return operation;
+        }
+        catch (IOException e) {
+            throw new IllegalArgumentException("failed parsing document", e);
+        }
+    }
+
     /** Returns the next document operation, or null if we have reached the end */
     public DocumentOperation next() {
         switch (state) {
diff --git a/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java b/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java
new file mode 100644
index 00000000000..0fbdd0b28c7
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java
@@ -0,0 +1,64 @@
+package com.yahoo.document.json;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+
+/**
+ * A {@link TokenBuffer} which only buffers tokens when needed, i.e., when peeking.
+ *
+ * @author jonmv
+ */
+public class LazyTokenBuffer extends TokenBuffer {
+
+    private final JsonParser parser;
+
+    public LazyTokenBuffer(JsonParser parser) {
+        this.parser = parser;
+        try { addFromParser(parser); }
+        catch (IOException e) { throw new IllegalArgumentException("failed parsing document JSON", e); }
+        if (JsonToken.START_OBJECT != current())
+            throw new IllegalArgumentException("expected start of JSON object, but got " + current());
+        updateNesting(current());
+    }
+
+    void advance() {
+        super.advance();
+        if (tokens.isEmpty() && nesting() > 0) tokens.add(nextToken()); // Fill current token if needed and possible.
+    }
+
+    @Override
+    public Supplier<Token> lookahead() {
+        return new Supplier<>() {
+            int localNesting = nesting();
+            Supplier<Token> buffered = LazyTokenBuffer.super.lookahead();
+            @Override public Token get() {
+                if (localNesting == 0)
+                    return null;
+
+                Token token = buffered.get();
+                if (token == null) {
+                    token = nextToken();
+                    tokens.add(token);
+                }
+                localNesting += nestingOffset(token.token);
+                return token;
+            }
+        };
+    }
+
+    private Token nextToken() {
+        try {
+            JsonToken token = parser.nextValue();
+            if (token == null)
+                throw new IllegalStateException("no more JSON tokens");
+            return new Token(token, parser.getCurrentName(), parser.getText());
+        }
+        catch (IOException e) {
+            throw new IllegalArgumentException("failed reading document JSON", e);
+        }
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java
index dec84e46b77..3a48f71c4cd 100644
--- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java
+++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java
@@ -1,15 +1,16 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.document.json;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonToken;
 import com.google.common.base.Preconditions;
 
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.function.Supplier;
+
 /**
  * Helper class to enable lookahead in the token stream.
  *
@@ -17,101 +18,76 @@ import com.google.common.base.Preconditions;
  */
 public class TokenBuffer {
 
-    private final List<Token> tokens;
+    final Deque<Token> tokens = new ArrayDeque<>();
 
-    private int position = 0;
     private int nesting = 0;
 
-    public TokenBuffer() {
-        this(new ArrayList<>());
-    }
-
-    public TokenBuffer(List<Token> tokens) {
-        this.tokens = tokens;
-        if (tokens.size() > 0)
-            updateNesting(tokens.get(position).token);
-    }
+    public TokenBuffer() { }
 
     /** Returns whether any tokens are available in this */
-    public boolean isEmpty() { return remaining() == 0; }
-
-    public JsonToken previous() {
-        updateNestingGoingBackwards(current());
-        position--;
-        return current();
-    }
-
-    /** Returns the current token without changing position, or null if none */
-    public JsonToken current() {
-        if (isEmpty()) return null;
-        Token token = tokens.get(position);
-        if (token == null) return null;
-        return token.token;
-    }
+    public boolean isEmpty() { return tokens.isEmpty(); }
 
+    /** Returns the next token, or null, and updates the nesting count of this. */
     public JsonToken next() {
-        position++;
+        advance();
         JsonToken token = current();
         updateNesting(token);
         return token;
     }
 
-    /** Returns a given number of tokens ahead, or null if none */
-    public JsonToken peek(int ahead) {
-        if (tokens.size() <= position + ahead) return null;
-        return tokens.get(position + ahead).token;
+    void advance() {
+        tokens.poll();
+    }
+
+    /** Returns the current token without changing position, or null if none */
+    public JsonToken current() {
+        return isEmpty() ? null : tokens.peek().token;
     }
 
     /** Returns the current token name without changing position, or null if none */
     public String currentName() {
-        if (isEmpty()) return null;
-        Token token = tokens.get(position);
-        if (token == null) return null;
-        return token.name;
+        return isEmpty() ? null : tokens.peek().name;
     }
 
     /** Returns the current token text without changing position, or null if none */
     public String currentText() {
-        if (isEmpty()) return null;
-        Token token = tokens.get(position);
-        if (token == null) return null;
-        return token.text;
+        return isEmpty() ? null : tokens.peek().text;
     }
 
-    public int remaining() {
-        return tokens.size() - position;
+    /**
+     * Returns a sequence of remaining tokens in this, or nulls when none remain.
+     * This may fill the token buffer, but not otherwise modify it.
+     */
+    public Supplier<Token> lookahead() {
+        Iterator<Token> iterator = tokens.iterator();
+        if (iterator.hasNext()) iterator.next();
+        return () -> iterator.hasNext() ? iterator.next() : null;
     }
 
     private void add(JsonToken token, String name, String text) {
-        tokens.add(tokens.size(), new Token(token, name, text));
+        tokens.add(new Token(token, name, text));
     }
 
-    public void bufferObject(JsonToken first, JsonParser tokens) {
-        bufferJsonStruct(first, tokens, JsonToken.START_OBJECT);
+    public void bufferObject(JsonParser parser) {
+        bufferJsonStruct(parser, JsonToken.START_OBJECT);
     }
 
-    private void bufferJsonStruct(JsonToken first, JsonParser tokens, JsonToken firstToken) {
-        int localNesting = 0;
-        JsonToken t = first;
+    private void bufferJsonStruct(JsonParser parser, JsonToken firstToken) {
+        JsonToken token = parser.currentToken();
+        Preconditions.checkArgument(token == firstToken,
+                                    "Expected %s, got %s.", firstToken.name(), token);
+        updateNesting(token);
 
-        Preconditions.checkArgument(first == firstToken,
-                "Expected %s, got %s.", firstToken.name(), t);
-        if (remaining() == 0) {
-            updateNesting(t);
+        try {
+            for (int nesting = addFromParser(parser); nesting > 0; nesting += addFromParser(parser))
+                parser.nextValue();
         }
-        localNesting = storeAndPeekNesting(t, localNesting, tokens);
-        while (localNesting > 0) {
-            t = nextValue(tokens);
-            localNesting = storeAndPeekNesting(t, localNesting, tokens);
+        catch (IOException e) {
+            throw new IllegalArgumentException(e);
         }
     }
 
-    private int storeAndPeekNesting(JsonToken t, int nesting, JsonParser tokens) {
-        addFromParser(t, tokens);
-        return nesting + nestingOffset(t);
-    }
-
-    private int nestingOffset(JsonToken token) {
+    int nestingOffset(JsonToken token) {
         if (token == null) return 0;
         if (token.isStructStart()) {
             return 1;
@@ -122,43 +98,23 @@ public class TokenBuffer {
         }
     }
 
-    private void addFromParser(JsonToken t, JsonParser tokens) {
-        try {
-            add(t, tokens.getCurrentName(), tokens.getText());
-        } catch (IOException e) {
-            throw new IllegalArgumentException(e);
-        }
-    }
-
-    private JsonToken nextValue(JsonParser tokens) {
-        try {
-            return tokens.nextValue();
-        } catch (IOException e) {
-            throw new IllegalArgumentException(e);
-        }
+    int addFromParser(JsonParser tokens) throws IOException {
+        add(tokens.currentToken(), tokens.getCurrentName(), tokens.getText());
+        return nestingOffset(tokens.currentToken());
     }
 
-    private void updateNesting(JsonToken token) {
+    void updateNesting(JsonToken token) {
         nesting += nestingOffset(token);
     }
 
-    private void updateNestingGoingBackwards(JsonToken token) {
-        nesting -= nestingOffset(token);
-    }
-
     public int nesting() {
         return nesting;
     }
 
     public void skipToRelativeNesting(int relativeNesting) {
         int initialNesting = nesting();
-        do {
-            next();
-        } while ( nesting() > initialNesting + relativeNesting);
-    }
-
-    public List<Token> rest() {
-        return tokens.subList(position, tokens.size());
+        do next();
+        while (nesting() > initialNesting + relativeNesting);
     }
 
     public static final class Token {
diff --git a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java
index 74656762fe1..aef7e1cffe2 100644
--- a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java
+++ b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java
@@ -86,16 +86,6 @@ public class DocumentParser {
     private void handleIdentLevelOne(DocumentParseInfo documentParseInfo, boolean docIdAndOperationIsSetExternally)
             throws IOException {
         JsonToken currentToken = parser.getCurrentToken();
-        if (currentToken == JsonToken.VALUE_TRUE || currentToken == JsonToken.VALUE_FALSE) {
-            try {
-                if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) {
-                    documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue());
-                    return;
-                }
-            } catch (IOException e) {
-                throw new RuntimeException("Got IO exception while parsing document", e);
-            }
-        }
         if ((currentToken == JsonToken.VALUE_TRUE || currentToken == JsonToken.VALUE_FALSE) &&
                 CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) {
             documentParseInfo.create = Optional.of(currentToken == JsonToken.VALUE_TRUE);
@@ -111,12 +101,11 @@ public class DocumentParser {
         }
     }
 
-    private  void handleIdentLevelTwo(DocumentParseInfo documentParseInfo) {
+    private void handleIdentLevelTwo(DocumentParseInfo documentParseInfo) {
         try {
-            JsonToken currentToken = parser.getCurrentToken();
             // "fields" opens a dictionary and is therefore on level two which might be surprising.
-            if (currentToken == JsonToken.START_OBJECT && FIELDS.equals(parser.getCurrentName())) {
-                documentParseInfo.fieldsBuffer.bufferObject(currentToken, parser);
+            if (parser.currentToken() == JsonToken.START_OBJECT && FIELDS.equals(parser.getCurrentName())) {
+                documentParseInfo.fieldsBuffer.bufferObject(parser);
                 processIndent();
             }
         } catch (IOException e) {
diff --git a/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java
index 2dce07cdbe6..e859306f04d 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java
@@ -8,6 +8,7 @@ import com.yahoo.document.json.TokenBuffer;
 import java.util.Optional;
 
 public class DocumentParseInfo {
+    public DocumentParseInfo() { }
     public DocumentId documentId;
     public Optional<Boolean> create = Optional.empty();
     public Optional<String> condition = Optional.empty();
diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java
index 0b7b1ae9996..1fd4029b1a5 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java
@@ -4,13 +4,15 @@ package com.yahoo.document.json.readers;
 import com.fasterxml.jackson.core.JsonToken;
 import com.yahoo.document.datatypes.TensorFieldValue;
 import com.yahoo.document.json.TokenBuffer;
-import com.yahoo.slime.Inspector;
-import com.yahoo.slime.Type;
+import com.yahoo.document.json.TokenBuffer.Token;
 import com.yahoo.tensor.IndexedTensor;
 import com.yahoo.tensor.MixedTensor;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorAddress;
 import com.yahoo.tensor.TensorType;
+import com.yahoo.tensor.TensorType.Dimension;
+
+import java.util.function.Supplier;
 
 import static com.yahoo.document.json.readers.JsonParserHelpers.*;
 import static com.yahoo.tensor.serialization.JsonFormat.decodeHexString;
@@ -37,36 +39,43 @@ public class TensorReader {
         Tensor.Builder builder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType());
         expectOneOf(buffer.current(), JsonToken.START_OBJECT, JsonToken.START_ARRAY);
         int initNesting = buffer.nesting();
-        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
-            if (TENSOR_CELLS.equals(buffer.currentName()) && ! primitiveContent(buffer)) {
+        while (true) {
+            Supplier<Token> lookahead = buffer.lookahead();
+            Token next = lookahead.get();
+            if (TENSOR_CELLS.equals(next.name) && ! primitiveContent(next.token, lookahead.get().token)) {
+                buffer.next();
                 readTensorCells(buffer, builder);
             }
-            else if (TENSOR_VALUES.equals(buffer.currentName()) && builder.type().dimensions().stream().allMatch(d -> d.isIndexed())) {
+            else if (TENSOR_VALUES.equals(next.name) && builder.type().dimensions().stream().allMatch(Dimension::isIndexed)) {
+                buffer.next();
                 readTensorValues(buffer, builder);
             }
-            else if (TENSOR_BLOCKS.equals(buffer.currentName())) {
+            else if (TENSOR_BLOCKS.equals(next.name)) {
+                buffer.next();
                 readTensorBlocks(buffer, builder);
             }
-            else if (TENSOR_TYPE.equals(buffer.currentName()) && buffer.current() == JsonToken.VALUE_STRING) {
+            else if (TENSOR_TYPE.equals(next.name) && next.token == JsonToken.VALUE_STRING) {
+                buffer.next();
                 // Ignore input tensor type
             }
+            else if (buffer.nesting() == initNesting && JsonToken.END_OBJECT == next.token) {
+                buffer.next();
+                break;
+            }
             else {
-                buffer.previous(); // Back up to the start of the enclosing block
                 readDirectTensorValue(buffer, builder);
-                buffer.previous(); // ... and back up to the end of the enclosing block
+                break;
             }
         }
         expectOneOf(buffer.current(), JsonToken.END_OBJECT, JsonToken.END_ARRAY);
         tensorFieldValue.assign(builder.build());
     }
 
-    static boolean primitiveContent(TokenBuffer buffer) {
-        JsonToken cellsValue = buffer.current();
-        if (cellsValue.isScalarValue()) return true;
-        if (cellsValue == JsonToken.START_ARRAY) {
-            JsonToken firstArrayValue = buffer.peek(1);
-            if (firstArrayValue == JsonToken.END_ARRAY) return false;
-            if (firstArrayValue.isScalarValue()) return true;
+    static boolean primitiveContent(JsonToken current, JsonToken next) {
+        if (current.isScalarValue()) return true;
+        if (current == JsonToken.START_ARRAY) {
+            if (next == JsonToken.END_ARRAY) return false;
+            if (next.isScalarValue()) return true;
         }
         return false;
     }
@@ -186,7 +195,7 @@ public class TensorReader {
         boolean hasIndexed = builder.type().dimensions().stream().anyMatch(TensorType.Dimension::isIndexed);
         boolean hasMapped = builder.type().dimensions().stream().anyMatch(TensorType.Dimension::isMapped);
 
-        if (isArrayOfObjects(buffer, 0))
+        if (isArrayOfObjects(buffer))
             readTensorCells(buffer, builder);
         else if ( ! hasMapped)
             readTensorValues(buffer, builder);
@@ -196,10 +205,12 @@ public class TensorReader {
             readTensorCells(buffer, builder);
     }
 
-    private static boolean isArrayOfObjects(TokenBuffer buffer, int ahead) {
-        if (buffer.peek(ahead++) != JsonToken.START_ARRAY) return false;
-        if (buffer.peek(ahead) == JsonToken.START_ARRAY) return isArrayOfObjects(buffer, ahead); // nested array
-        return buffer.peek(ahead) == JsonToken.START_OBJECT;
+    private static boolean isArrayOfObjects(TokenBuffer buffer) {
+        if (buffer.current() != JsonToken.START_ARRAY) return false;
+        Supplier<Token> lookahead = buffer.lookahead();
+        Token next;
+        while ((next = lookahead.get()).token == JsonToken.START_ARRAY) { }
+        return next.token == JsonToken.START_OBJECT;
     }
 
     private static TensorAddress readAddress(TokenBuffer buffer, TensorType type) {
diff --git a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java
index 113b8732b23..c7303d31ea2 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java
@@ -238,7 +238,7 @@ public class VespaJsonDocumentReader {
                                  "Expected end of JSON struct (%s), got %s", expectedFinalToken, buffer.current());
         Preconditions.checkState(buffer.nesting() == 0, "Nesting not zero at end of operation");
         Preconditions.checkState(buffer.next() == null, "Dangling data at end of operation");
-        Preconditions.checkState(buffer.remaining() == 0, "Dangling data at end of operation");
+        Preconditions.checkState(buffer.isEmpty(), "Dangling data at end of operation");
     }
 
 }
diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
index 5a9f02c790d..080528fea77 100644
--- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
+++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
@@ -2120,69 +2120,93 @@ public class JsonReaderTestCase {
     @Test
     public void tensor_modify_update_with_replace_operation() {
         assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.REPLACE, "sparse_tensor",
-                inputJson("{",
-                        "  'operation': 'replace',",
-                        "  'cells': [",
-                        "    { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}"));
+                                 """
+                                 {
+                                   "operation": "replace",
+                                   "cells": [
+                                     { "address": { "x": "a", "y": "b" }, "value": 2.0 }
+                                   ]
+                                 }""");
     }
 
     @Test
     public void tensor_modify_update_with_add_operation() {
         assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.ADD, "sparse_tensor",
-                inputJson("{",
-                        "  'operation': 'add',",
-                        "  'cells': [",
-                        "    { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}"));
+                                 """
+                                 {
+                                   "operation": "add",
+                                   "cells": [
+                                     { "address": { "x": "a", "y": "b" }, "value": 2.0 }
+                                   ]
+                                 }""");
     }
 
     @Test
     public void tensor_modify_update_with_multiply_operation() {
         assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.MULTIPLY, "sparse_tensor",
-                inputJson("{",
-                        "  'operation': 'multiply',",
-                        "  'cells': [",
-                        "    { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}"));
+                                 """
+                                 {
+                                   "operation": "multiply",
+                                   "cells": [
+                                     { "address": { "x": "a", "y": "b" }, "value": 2.0 }
+                                   ]
+                                 }""");
     }
 
     @Test
     public void tensor_modify_update_with_create_non_existing_cells_true() {
         assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.ADD, true, "sparse_tensor",
-                inputJson("{",
-                        "  'operation': 'add',",
-                        "  'create': true,",
-                        "  'cells': [",
-                        "    { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}"));
+                                 """
+                                 {
+                                   "operation": "add",
+                                   "create": true,
+                                   "cells": [
+                                     { "address": { "x": "a", "y": "b" }, "value": 2.0 }
+                                   ]
+                                 }""");
     }
 
     @Test
     public void tensor_modify_update_with_create_non_existing_cells_false() {
         assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.ADD, false, "sparse_tensor",
-                inputJson("{",
-                        "  'operation': 'add',",
-                        "  'create': false,",
-                        "  'cells': [",
-                        "    { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}"));
+                                 """
+                                 {
+                                   "operation": "add",
+                                   "create": false,
+                                   "cells": [
+                                     { "address": { "x": "a", "y": "b" }, "value": 2.0 }
+                                   ]
+                                 }""");
     }
 
     @Test
     public void tensor_modify_update_treats_the_input_tensor_as_sparse() {
         // Note that the type of the tensor in the modify update is sparse (it only has mapped dimensions).
         assertTensorModifyUpdate("tensor(x{},y{}):{{x:0,y:0}:2.0, {x:1,y:2}:3.0}",
-                TensorModifyUpdate.Operation.REPLACE, "dense_tensor",
-                inputJson("{",
-                        "  'operation': 'replace',",
-                        "  'cells': [",
-                        "    { 'address': { 'x': '0', 'y': '0' }, 'value': 2.0 },",
-                        "    { 'address': { 'x': '1', 'y': '2' }, 'value': 3.0 } ]}"));
+                                 TensorModifyUpdate.Operation.REPLACE, "dense_tensor",
+                                 """
+                                 {
+                                   "operation": "replace",
+                                   "cells": [
+                                     { "address": { "x": "0", "y": "0" }, "value": 2.0 },
+                                     { "address": { "x": "1", "y": "2" }, "value": 3.0 }
+                                   ]
+                                 }""");
     }
 
     @Test
     public void tensor_modify_update_on_non_tensor_field_throws() {
         try {
-            JsonReader reader = createReader(inputJson("{ 'update': 'id:unittest:smoke::doc1',",
-                                                       "  'fields': {",
-                                                       "    'something': {",
-                                                       "      'modify': {} }}}"));
+            JsonReader reader = createReader("""
+                                             {
+                                               "update": "id:unittest:smoke::doc1",
+                                               "fields": {
+                                                 "something": {
+                                                   "modify": {}
+                                                 }
+                                               }
+                                             }
+                                             """);
             reader.readSingleDocument(DocumentOperationType.UPDATE, "id:unittest:smoke::doc1");
             fail("Expected exception");
         }
@@ -2196,95 +2220,125 @@ public class JsonReaderTestCase {
     public void tensor_modify_update_on_dense_unbound_tensor_throws() {
         illegalTensorModifyUpdate("Error in 'dense_unbound_tensor': A modify update cannot be applied to tensor types with indexed unbound dimensions. Field 'dense_unbound_tensor' has unsupported tensor type 'tensor(x[],y[])'",
                                   "dense_unbound_tensor",
-                                  "{",
-                                  "  'operation': 'replace',",
-                                  "  'cells': [",
-                                  "    { 'address': { 'x': '0', 'y': '0' }, 'value': 2.0 } ]}");
+                                  """
+                                  {
+                                    "operation": "replace",
+                                    "cells": [
+                                      { "address": { "x": "0", "y": "0" }, "value": 2.0 }
+                                    ]
+                                  }""");
     }
 
     @Test
     public void tensor_modify_update_on_sparse_tensor_with_single_dimension_short_form() {
-        assertTensorModifyUpdate("{{x:a}:2.0, {x:c}: 3.0}",  TensorModifyUpdate.Operation.REPLACE, "sparse_single_dimension_tensor",
-                                 inputJson("{",
-                                           "  'operation': 'replace',",
-                                           "  'cells': {",
-                                           "    'a': 2.0,",
-                                           "    'c': 3.0 }}"));
+        assertTensorModifyUpdate("{{x:a}:2.0, {x:c}: 3.0}", TensorModifyUpdate.Operation.REPLACE, "sparse_single_dimension_tensor",
+                                 """
+                                 {
+                                   "operation": "replace",
+                                   "cells": {
+                                     "a": 2.0,
+                                     "c": 3.0
+                                   }
+                                 }""");
     }
 
     @Test
     public void tensor_modify_update_with_replace_operation_mixed() {
         assertTensorModifyUpdate("{{x:a,y:0}:2.0}", TensorModifyUpdate.Operation.REPLACE, "mixed_tensor",
-                inputJson("{",
-                        "  'operation': 'replace',",
-                        "  'cells': [",
-                        "    { 'address': { 'x': 'a', 'y': '0' }, 'value': 2.0 } ]}"));
+                                 """
+                                 {
+                                   "operation": "replace",
+                                   "cells": [
+                                     { "address": { "x": "a", "y": "0" }, "value": 2.0 }
+                                   ]
+                                 }""");
     }
 
     @Test
     public void tensor_modify_update_with_replace_operation_mixed_block_short_form_array() {
         assertTensorModifyUpdate("{{x:a,y:0}:1,{x:a,y:1}:2,{x:a,y:2}:3}", TensorModifyUpdate.Operation.REPLACE, "mixed_tensor",
-                                 inputJson("{",
-                                           "  'operation': 'replace',",
-                                           "  'blocks': [",
-                                           "    { 'address': { 'x': 'a' }, 'values': [1,2,3] } ]}"));
+                                 """
+                                 {
+                                   "operation": "replace",
+                                   "blocks": [
+                                     { "address": { "x": "a" }, "values": [1,2,3] }
+                                   ]
+                                 }""");
     }
 
     @Test
     public void tensor_modify_update_with_replace_operation_mixed_block_short_form_must_specify_full_subspace() {
         illegalTensorModifyUpdate("Error in 'mixed_tensor': At {x:a}: Expected 3 values, but got 2",
-                                   "mixed_tensor",
-                                  inputJson("{",
-                                            "  'operation': 'replace',",
-                                            "  'blocks': {",
-                                            "    'a': [2,3] } }"));
+                                  "mixed_tensor",
+                                  """
+                                  {
+                                    "operation": "replace",
+                                    "blocks": {
+                                      "a": [2,3]
+                                    }
+                                  }""");
     }
 
     @Test
     public void tensor_modify_update_with_replace_operation_mixed_block_short_form_map() {
         assertTensorModifyUpdate("{{x:a,y:0}:1,{x:a,y:1}:2,{x:a,y:2}:3}", TensorModifyUpdate.Operation.REPLACE, "mixed_tensor",
-                                 inputJson("{",
-                                           "  'operation': 'replace',",
-                                           "  'blocks': {",
-                                           "    'a': [1,2,3] } }"));
+                                 """
+                                 {
+                                   "operation": "replace",
+                                   "blocks": {
+                                     "a": [1,2,3]
+                                   }
+                                 }""");
     }
 
     @Test
     public void tensor_modify_update_with_add_operation_mixed() {
         assertTensorModifyUpdate("{{x:a,y:0}:2.0}", TensorModifyUpdate.Operation.ADD, "mixed_tensor",
-                inputJson("{",
-                        "  'operation': 'add',",
-                        "  'cells': [",
-                        "    { 'address': { 'x': 'a', 'y': '0' }, 'value': 2.0 } ]}"));
+                                 """
+                                 {
+                                   "operation": "add",
+                                   "cells": [
+                                     { "address": { "x": "a", "y": "0" }, "value": 2.0 }
+                                   ]
+                                 }""");
     }
 
     @Test
     public void tensor_modify_update_with_multiply_operation_mixed() {
         assertTensorModifyUpdate("{{x:a,y:0}:2.0}", TensorModifyUpdate.Operation.MULTIPLY, "mixed_tensor",
-                inputJson("{",
-                        "  'operation': 'multiply',",
-                        "  'cells': [",
-                        "    { 'address': { 'x': 'a', 'y': '0' }, 'value': 2.0 } ]}"));
+                                 """
+                                 {
+                                   "operation": "multiply",
+                                   "cells": [
+                                     { "address": { "x": "a", "y": "0" }, "value": 2.0 }
+                                   ]
+                                 }""");
     }
 
     @Test
     public void tensor_modify_update_with_out_of_bound_cells_throws() {
         illegalTensorModifyUpdate("Error in 'dense_tensor': Dimension 'y' has label '3' but type is tensor(x[2],y[3])",
                                   "dense_tensor",
-                                  "{",
-                                  "  'operation': 'replace',",
-                                  "  'cells': [",
-                                  "    { 'address': { 'x': '0', 'y': '3' }, 'value': 2.0 } ]}");
+                                  """
+                                  {
+                                    "operation": "replace",
+                                    "cells": [
+                                      { "address": { "x": "0", "y": "3" }, "value": 2.0 }
+                                    ]
+                                  }""");
     }
 
     @Test
     public void tensor_modify_update_with_out_of_bound_cells_throws_mixed() {
         illegalTensorModifyUpdate("Error in 'mixed_tensor': Dimension 'y' has label '3' but type is tensor(x{},y[3])",
                                   "mixed_tensor",
-                                  "{",
-                                  "  'operation': 'replace',",
-                                  "  'cells': [",
-                                  "    { 'address': { 'x': '0', 'y': '3' }, 'value': 2.0 } ]}");
+                                  """
+                                  {
+                                    "operation": "replace",
+                                    "cells": [
+                                      { "address": { "x": "0", "y": "3" }, "value": 2.0 }
+                                    ]
+                                  }""");
     }
 
 
@@ -2292,87 +2346,113 @@ public class JsonReaderTestCase {
     public void tensor_modify_update_with_unknown_operation_throws() {
         illegalTensorModifyUpdate("Error in 'sparse_tensor': Unknown operation 'unknown' in modify update for field 'sparse_tensor'",
                                   "sparse_tensor",
-                                  "{",
-                                  "  'operation': 'unknown',",
-                                   "  'cells': [",
-                                   "    { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}");
+                                  """
+                                  {
+                                    "operation": "unknown",
+                                    "cells": [
+                                      { "address": { "x": "a", "y": "b" }, "value": 2.0 }
+                                    ]
+                                  }""");
     }
 
     @Test
     public void tensor_modify_update_without_operation_throws() {
         illegalTensorModifyUpdate("Error in 'sparse_tensor': Modify update for field 'sparse_tensor' does not contain an operation",
                                   "sparse_tensor",
-                                  "{",
-                                  "  'cells': [] }");
+                                  """
+                                  {
+                                    "cells": []
+                                  }""");
     }
 
     @Test
     public void tensor_modify_update_without_cells_throws() {
         illegalTensorModifyUpdate("Error in 'sparse_tensor': Modify update for field 'sparse_tensor' does not contain tensor cells",
                                   "sparse_tensor",
-                                  "{",
-                                  "  'operation': 'replace' }");
+                                  """
+                                  {
+                                    "operation": "replace"
+                                  }""");
     }
 
     @Test
     public void tensor_modify_update_with_unknown_content_throws() {
         illegalTensorModifyUpdate("Error in 'sparse_tensor': Unknown JSON string 'unknown' in modify update for field 'sparse_tensor'",
                                   "sparse_tensor",
-                                  "{",
-                                  "  'unknown': 'here' }");
+                                  """
+                                  {
+                                    "unknown": "here"
+                                  }""");
     }
 
     @Test
     public void tensor_add_update_on_sparse_tensor() {
         assertTensorAddUpdate("{{x:a,y:b}:2.0, {x:c,y:d}: 3.0}", "sparse_tensor",
-                inputJson("{",
-                        "  'cells': [",
-                        "    { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 },",
-                        "    { 'address': { 'x': 'c', 'y': 'd' }, 'value': 3.0 } ]}"));
+                              """
+                              {
+                                "cells": [
+                                  { "address": { "x": "a", "y": "b" }, "value": 2.0 },
+                                  { "address": { "x": "c", "y": "d" }, "value": 3.0 }
+                                ]
+                              }""");
     }
 
     @Test
     public void tensor_add_update_on_sparse_tensor_with_single_dimension_short_form() {
         assertTensorAddUpdate("{{x:a}:2.0, {x:c}: 3.0}", "sparse_single_dimension_tensor",
-                              inputJson("{",
-                                        "  'cells': {",
-                                        "    'a': 2.0,",
-                                        "    'c': 3.0 }}"));
+                              """
+                              {
+                                "cells": {
+                                  "a": 2.0,
+                                  "c": 3.0
+                                }
+                              }""");
     }
 
     @Test
     public void tensor_add_update_on_mixed_tensor() {
         assertTensorAddUpdate("{{x:a,y:0}:2.0, {x:a,y:1}:3.0, {x:a,y:2}:0.0}", "mixed_tensor",
-                inputJson("{",
-                        "  'cells': [",
-                        "    { 'address': { 'x': 'a', 'y': '0' }, 'value': 2.0 },",
-                        "    { 'address': { 'x': 'a', 'y': '1' }, 'value': 3.0 } ]}"));
+                              """
+                              {
+                                "cells": [
+                                  { "address": { "x": "a", "y": "0" }, "value": 2.0 },
+                                  { "address": { "x": "a", "y": "1" }, "value": 3.0 }
+                                ]
+                              }""");
     }
 
     @Test
     public void tensor_add_update_on_mixed_with_out_of_bound_dense_cells_throws() {
         illegalTensorAddUpdate("Error in 'mixed_tensor': Index 3 out of bounds for length 3",
                                "mixed_tensor",
-                               "{",
-                               "  'cells': [",
-                               "    { 'address': { 'x': '0', 'y': '3' }, 'value': 2.0 } ]}");
+                               """
+                               {
+                                 "cells": [
+                                   { "address": { "x": "0", "y": "3" }, "value": 2.0 }
+                                 ]
+                               }""");
     }
 
     @Test
     public void tensor_add_update_on_dense_tensor_throws() {
         illegalTensorAddUpdate("Error in 'dense_tensor': An add update can only be applied to tensors with at least one sparse dimension. Field 'dense_tensor' has unsupported tensor type 'tensor(x[2],y[3])'",
                                "dense_tensor",
-                               "{",
-                               "  'cells': [] }");
+                               """
+                               {
+                                 "cells": [ ]
+                               }""");
     }
 
     @Test
     public void tensor_add_update_on_not_fully_specified_cell_throws() {
         illegalTensorAddUpdate("Error in 'sparse_tensor': Missing a label for dimension 'y' for tensor(x{},y{})",
                                "sparse_tensor",
-                               "{",
-                               "  'cells': [",
-                               "    { 'address': { 'x': 'a' }, 'value': 2.0 } ]}");
+                               """
+                               {
+                                 "cells": [
+                                   { "address": { "x": "a" }, "value": 2.0 }
+                                 ]
+                               }""");
     }
 
     @Test
@@ -2388,146 +2468,176 @@ public class JsonReaderTestCase {
     @Test
     public void tensor_remove_update_on_sparse_tensor() {
         assertTensorRemoveUpdate("{{x:a,y:b}:1.0,{x:c,y:d}:1.0}", "sparse_tensor",
-                inputJson("{",
-                        "  'addresses': [",
-                        "    { 'x': 'a', 'y': 'b' },",
-                        "    { 'x': 'c', 'y': 'd' } ]}"));
+                                 """
+                                 {
+                                   "addresses": [
+                                     { "x": "a", "y": "b" },
+                                     { "x": "c", "y": "d" }
+                                   ]
+                                 }""");
     }
 
     @Test
     public void tensor_remove_update_on_mixed_tensor() {
         assertTensorRemoveUpdate("{{x:1}:1.0,{x:2}:1.0}", "mixed_tensor",
-                inputJson("{",
-                        "  'addresses': [",
-                        "    { 'x': '1' },",
-                        "    { 'x': '2' } ]}"));
+                                 """
+                                 {
+                                   "addresses": [
+                                     { "x": "1" },
+                                     { "x": "2" }
+                                   ]
+                                 }""");
     }
 
     @Test
     public void tensor_remove_update_on_sparse_tensor_with_not_fully_specified_address() {
         assertTensorRemoveUpdate("{{y:b}:1.0,{y:d}:1.0}", "sparse_tensor",
-                inputJson("{",
-                        "  'addresses': [",
-                        "    { 'y': 'b' },",
-                        "    { 'y': 'd' } ]}"));
+                                 """
+                                 {
+                                   "addresses": [
+                                     { "y": "b" },
+                                     { "y": "d" }
+                                   ]
+                                 }""");
     }
 
     @Test
     public void tensor_remove_update_on_mixed_tensor_with_not_fully_specified_address() {
         assertTensorRemoveUpdate("{{x:1,z:a}:1.0,{x:2,z:b}:1.0}", "mixed_tensor_adv",
-                inputJson("{",
-                        "  'addresses': [",
-                        "    { 'x': '1', 'z': 'a' },",
-                        "    { 'x': '2', 'z': 'b' } ]}"));
+                                 """
+                                 {
+                                   "addresses": [
+                                     { "x": "1", "z": "a" },
+                                     { "x": "2", "z": "b" }
+                                   ]
+                                 }""");
     }
 
     @Test
     public void tensor_remove_update_on_mixed_tensor_with_dense_addresses_throws() {
         illegalTensorRemoveUpdate("Error in 'mixed_tensor': Indexed dimension address 'y' should not be specified in remove update",
                                   "mixed_tensor",
-                                  "{",
-                                  "  'addresses': [",
-                                  "    { 'x': '1', 'y': '0' },",
-                                  "    { 'x': '2', 'y': '0' } ]}");
+                                  """
+                                  {
+                                    "addresses": [
+                                      { "x": "1", "y": "0" },
+                                      { "x": "2", "y": "0" }
+                                    ]
+                                  }""");
     }
 
     @Test
     public void tensor_remove_update_on_dense_tensor_throws() {
         illegalTensorRemoveUpdate("Error in 'dense_tensor': A remove update can only be applied to tensors with at least one sparse dimension. Field 'dense_tensor' has unsupported tensor type 'tensor(x[2],y[3])'",
                                   "dense_tensor",
-                                  "{",
-                                  "  'addresses': [] }");
+                                  """
+                                  {
+                                    "addresses": []
+                                  }""");
     }
 
     @Test
     public void tensor_remove_update_with_stray_dimension_throws() {
         illegalTensorRemoveUpdate("Error in 'sparse_tensor': tensor(x{},y{}) does not contain dimension 'foo'",
-                "sparse_tensor",
-                "{",
-                "  'addresses': [",
-                "    { 'x': 'a', 'foo': 'b' } ]}");
+                                  "sparse_tensor",
+                                  """
+                                  {
+                                    "addresses": [
+                                      { "x": "a", "foo": "b" }
+                                    ]
+                                  }""");
 
         illegalTensorRemoveUpdate("Error in 'sparse_tensor': tensor(x{}) does not contain dimension 'foo'",
-                "sparse_tensor",
-                "{",
-                "  'addresses': [",
-                "    { 'x': 'c' },",
-                "    { 'x': 'a', 'foo': 'b' } ]}");
+                                  "sparse_tensor",
+                                  """
+                                  {
+                                    "addresses": [
+                                      { "x": "c" },
+                                      { "x": "a", "foo": "b" }
+                                    ]
+                                  }""");
     }
 
     @Test
     public void tensor_remove_update_without_cells_throws() {
         illegalTensorRemoveUpdate("Error in 'sparse_tensor': Remove update for field 'sparse_tensor' does not contain tensor addresses",
                                   "sparse_tensor",
-                                  "{'addresses': [] }");
+                                  """
+                                  {
+                                    "addresses": []
+                                  }""");
 
         illegalTensorRemoveUpdate("Error in 'mixed_tensor': Remove update for field 'mixed_tensor' does not contain tensor addresses",
                                   "mixed_tensor",
-                                  "{'addresses': [] }");
+                                  """
+                                  {
+                                    "addresses": []
+                                  }""");
     }
 
     @Test
     public void require_that_parser_propagates_datatype_parser_errors_predicate() {
         assertParserErrorMatches(
                 "Error in document 'id:unittest:testpredicate::0' - could not parse field 'boolean' of type 'predicate': " +
-                        "line 1:10 no viable alternative at character '>'",
-
-                "[",
-                "      {",
-                "          'fields': {",
-                "              'boolean': 'timestamp > 9000'",
-                "          },",
-                "          'put': 'id:unittest:testpredicate::0'",
-                "      }",
-                "]"
-        );
+                "line 1:10 no viable alternative at character '>'",
+                """
+                [
+                  {
+                    "fields": {
+                      "boolean": "timestamp > 9000"
+                    },
+                    "put": "id:unittest:testpredicate::0"
+                  }
+                ]
+                """);
     }
 
     @Test
     public void require_that_parser_propagates_datatype_parser_errors_string_as_int() {
         assertParserErrorMatches(
                 "Error in document 'id:unittest:testint::0' - could not parse field 'integerfield' of type 'int': " +
-                        "For input string: \" 1\"",
-
-                "[",
-                "      {",
-                "          'fields': {",
-                "              'integerfield': ' 1'",
-                "          },",
-                "          'put': 'id:unittest:testint::0'",
-                "      }",
-                "]"
-        );
+                "For input string: \" 1\"",
+                """
+                [
+                  {
+                    "fields": {
+                      "integerfield": " 1"
+                    },
+                    "put": "id:unittest:testint::0"
+                  }
+                ]
+                """);
     }
 
     @Test
     public void require_that_parser_propagates_datatype_parser_errors_overflowing_int() {
         assertParserErrorMatches(
                 "Error in document 'id:unittest:testint::0' - could not parse field 'integerfield' of type 'int': " +
-                        "For input string: \"281474976710656\"",
-
-                "[",
-                "      {",
-                "          'fields': {",
-                "              'integerfield': 281474976710656",
-                "          },",
-                "          'put': 'id:unittest:testint::0'",
-                "      }",
-                "]"
-        );
+                "For input string: \"281474976710656\"",
+                """
+                [
+                  {
+                    "fields": {
+                      "integerfield": 281474976710656
+                    },
+                    "put": "id:unittest:testint::0"
+                  }
+                ]
+                """);
     }
 
     @Test
     public void requireThatUnknownDocTypeThrowsIllegalArgumentException() {
-        final String jsonData = inputJson(
-                "[",
-                "      {",
-                "          'put': 'id:ns:walrus::walrus1',",
-                "          'fields': {",
-                "              'aField': 42",
-                "          }",
-                "      }",
-                "]");
+        String jsonData = """
+                          [
+                            {
+                              "put": "id:ns:walrus::walrus1",
+                              "fields": {
+                                "aField": 42
+                              }
+                            }
+                          ]
+                          """;
         try {
             new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next();
             fail();
@@ -2577,30 +2687,40 @@ public class JsonReaderTestCase {
         return createPutWithTensor(inputTensor, "sparse_tensor");
     }
     private DocumentPut createPutWithTensor(String inputTensor, String tensorFieldName) {
-        JsonReader reader = createReader(inputJson("[",
-                "{ 'put': '" + TENSOR_DOC_ID + "',",
-                "  'fields': {",
-                "    '" + tensorFieldName + "': " + inputTensor + "  }}]"));
-        return (DocumentPut) reader.next();
+        JsonReader streaming = createReader("""
+                                         {
+                                           "fields": {
+                                             "%s": %s
+                                           }
+                                         }
+                                         """.formatted(tensorFieldName, inputTensor));
+        DocumentPut lazyParsed = (DocumentPut) streaming.readSingleDocumentStreaming(DocumentOperationType.PUT, TENSOR_DOC_ID).operation();
+        JsonReader reader = createReader("""
+                                         [
+                                           {
+                                             "put": "%s",
+                                             "fields": {
+                                               "%s": %s
+                                             }
+                                           }
+                                         ]""".formatted(TENSOR_DOC_ID, tensorFieldName, inputTensor));
+        DocumentPut bufferParsed = (DocumentPut) reader.next();
+        assertEquals(lazyParsed, bufferParsed);
+        return bufferParsed;
     }
 
     private DocumentUpdate createAssignUpdateWithSparseTensor(String inputTensor) {
         return createAssignUpdateWithTensor(inputTensor, "sparse_tensor");
     }
     private DocumentUpdate createAssignUpdateWithTensor(String inputTensor, String tensorFieldName) {
-        JsonReader reader = createReader(inputJson("[",
-                "{ 'update': '" + TENSOR_DOC_ID + "',",
-                "  'fields': {",
-                "    '" + tensorFieldName + "': {",
-                "      'assign': " + (inputTensor != null ? inputTensor : "null") + " } } } ]"));
-        return (DocumentUpdate) reader.next();
+        return createTensorUpdate("assign", inputTensor, tensorFieldName);
     }
 
     private static Tensor assertSparseTensorField(String expectedTensor, DocumentPut put) {
         return assertTensorField(expectedTensor, put, "sparse_tensor");
     }
     private Tensor assertTensorField(String expectedTensor, String fieldName, String inputJson) {
-        return assertTensorField(expectedTensor, createPutWithTensor(inputJson, fieldName), fieldName);
+        return assertTensorField(expectedTensor, createPutWithTensor(inputJson(inputJson), fieldName), fieldName);
     }
     private static Tensor assertTensorField(String expectedTensor, DocumentPut put, String tensorFieldName) {
         return assertTensorField(Tensor.from(expectedTensor), put, tensorFieldName);
@@ -2673,12 +2793,29 @@ public class JsonReaderTestCase {
     }
 
     private DocumentUpdate createTensorUpdate(String operation, String tensorJson, String tensorFieldName) {
-        JsonReader reader = createReader(inputJson("[",
-                "{ 'update': '" + TENSOR_DOC_ID + "',",
-                "  'fields': {",
-                "    '" + tensorFieldName + "': {",
-                "      '" + operation + "': " + tensorJson + " }}}]"));
-        return (DocumentUpdate) reader.next();
+        JsonReader streaming = createReader("""
+                                            {
+                                              "fields": {
+                                                "%s": {
+                                                   "%s": %s
+                                                }
+                                              }
+                                            }""".formatted(tensorFieldName, operation, tensorJson));
+        DocumentUpdate lazyParsed = (DocumentUpdate) streaming.readSingleDocumentStreaming(DocumentOperationType.UPDATE, TENSOR_DOC_ID).operation();
+        JsonReader reader = createReader("""
+                                         [
+                                           {
+                                             "update": "%s",
+                                             "fields": {
+                                               "%s": {
+                                                 "%s": %s
+                                               }
+                                             }
+                                           }
+                                         ]""".formatted(TENSOR_DOC_ID, tensorFieldName, operation, tensorJson));
+        DocumentUpdate bufferParsed = (DocumentUpdate) reader.next();
+        assertEquals(lazyParsed, bufferParsed);
+        return bufferParsed;
     }
 
     private void assertTensorAddUpdate(String expectedTensor, String tensorFieldName, String tensorJson) {
diff --git a/document/src/test/java/com/yahoo/document/json/LazyTokenBufferTest.java b/document/src/test/java/com/yahoo/document/json/LazyTokenBufferTest.java
new file mode 100644
index 00000000000..3ed2ed531c3
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/json/LazyTokenBufferTest.java
@@ -0,0 +1,132 @@
+package com.yahoo.document.json;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.yahoo.document.json.TokenBuffer.Token;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * @author jonmv
+ */
+public class LazyTokenBufferTest {
+
+    @Test
+    public void testBuffer() throws IOException {
+        String json = """
+                      {
+                        "fields": {
+                          "foo": "bar",
+                          "baz": [1, 2, 3],
+                          "quu": { "qux": null }
+                        }
+                      }""";
+        JsonParser parser = new JsonFactory().createParser(json);
+        parser.nextValue();
+        parser.nextValue();
+        assertEquals(JsonToken.START_OBJECT, parser.currentToken());
+        assertEquals("fields", parser.currentName());
+
+        // Peeking through the buffer doesn't change nesting.
+        LazyTokenBuffer buffer = new LazyTokenBuffer(parser);
+        assertEquals(JsonToken.START_OBJECT, buffer.current());
+        assertEquals("fields", buffer.currentName());
+        assertEquals(1, buffer.nesting());
+
+        Supplier<Token> lookahead = buffer.lookahead();
+        Token peek = lookahead.get();
+        assertEquals(JsonToken.VALUE_STRING, peek.token);
+        assertEquals("foo", peek.name);
+        assertEquals("bar", peek.text);
+        assertEquals(1, buffer.nesting());
+
+        peek = lookahead.get();
+        assertEquals(JsonToken.START_ARRAY, peek.token);
+        assertEquals("baz", peek.name);
+        assertEquals(1, buffer.nesting());
+
+        peek = lookahead.get();
+        assertEquals(JsonToken.VALUE_NUMBER_INT, peek.token);
+        assertEquals("1", peek.text);
+
+        peek = lookahead.get();
+        assertEquals(JsonToken.VALUE_NUMBER_INT, peek.token);
+        assertEquals("2", peek.text);
+
+        peek = lookahead.get();
+        assertEquals(JsonToken.VALUE_NUMBER_INT, peek.token);
+        assertEquals("3", peek.text);
+
+        peek = lookahead.get();
+        assertEquals(JsonToken.END_ARRAY, peek.token);
+        assertEquals(1, buffer.nesting());
+
+        peek = lookahead.get();
+        assertEquals(JsonToken.START_OBJECT, peek.token);
+        assertEquals("quu", peek.name);
+        assertEquals(1, buffer.nesting());
+
+        peek = lookahead.get();
+        assertEquals(JsonToken.VALUE_NULL, peek.token);
+        assertEquals("qux", peek.name);
+
+        peek = lookahead.get();
+        assertEquals(JsonToken.END_OBJECT, peek.token);
+        assertEquals(1, buffer.nesting());
+
+        peek = lookahead.get();
+        assertEquals(JsonToken.END_OBJECT, peek.token);
+        assertEquals(1, buffer.nesting());
+
+        peek = lookahead.get();
+        assertNull(peek);
+
+        // Parser is now at the end.
+        assertEquals(JsonToken.END_OBJECT, parser.nextToken());
+        assertNull(parser.nextToken());
+
+        // Repeat iterating through the buffer, this time advancing it, and see that nesting changes.
+        assertEquals(JsonToken.VALUE_STRING, buffer.next());
+        assertEquals("foo", buffer.currentName());
+        assertEquals("bar", buffer.currentText());
+        assertEquals(1, buffer.nesting());
+
+        assertEquals(JsonToken.START_ARRAY, buffer.next());
+        assertEquals("baz", buffer.currentName());
+        assertEquals(2, buffer.nesting());
+
+        assertEquals(JsonToken.VALUE_NUMBER_INT, buffer.next());
+        assertEquals("1", buffer.currentText());
+
+        assertEquals(JsonToken.VALUE_NUMBER_INT, buffer.next());
+        assertEquals("2", buffer.currentText());
+
+        assertEquals(JsonToken.VALUE_NUMBER_INT, buffer.next());
+        assertEquals("3", buffer.currentText());
+
+        assertEquals(JsonToken.END_ARRAY, buffer.next());
+        assertEquals(1, buffer.nesting());
+
+        assertEquals(JsonToken.START_OBJECT, buffer.next());
+        assertEquals("quu", buffer.currentName());
+        assertEquals(2, buffer.nesting());
+
+        assertEquals(JsonToken.VALUE_NULL, buffer.next());
+        assertEquals("qux", buffer.currentName());
+
+        assertEquals(JsonToken.END_OBJECT, buffer.next());
+        assertEquals(1, buffer.nesting());
+
+        assertEquals(JsonToken.END_OBJECT, buffer.next());
+        assertEquals(0, buffer.nesting());
+
+        assertNull(buffer.next());
+    }
+
+}