From bc89d03da6c10eb38577c279cd26c82bf914a3bc Mon Sep 17 00:00:00 2001 From: jonmv Date: Fri, 26 Jan 2024 09:37:11 +0100 Subject: Revert "Merge pull request #30067 from vespa-engine/revert-30038-jonmv/leaner-token-buffer" This reverts commit b771fbe5fe648cf4c64e04341542e11c2e21cb9d, reversing changes made to 7b578506b5c4c59f4273e74af1f0db4a74f82175. --- .../java/com/yahoo/document/json/JsonReader.java | 51 +- .../com/yahoo/document/json/LazyTokenBuffer.java | 64 +++ .../java/com/yahoo/document/json/TokenBuffer.java | 140 ++---- .../document/json/document/DocumentParser.java | 17 +- .../document/json/readers/DocumentParseInfo.java | 1 + .../yahoo/document/json/readers/TensorReader.java | 53 +- .../json/readers/VespaJsonDocumentReader.java | 2 +- .../yahoo/document/json/JsonReaderTestCase.java | 533 +++++++++++++-------- .../yahoo/document/json/LazyTokenBufferTest.java | 132 +++++ .../restapi/resource/DocumentV1ApiHandler.java | 2 +- .../restapi/resource/DocumentV1ApiTest.java | 5 +- 11 files changed, 670 insertions(+), 330 deletions(-) create mode 100644 document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java create mode 100644 document/src/test/java/com/yahoo/document/json/LazyTokenBufferTest.java diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index 3e1743b8d45..08d1fe688ed 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -18,6 +18,7 @@ import java.io.InputStream; import java.util.Optional; import static com.yahoo.document.json.JsonReader.ReaderState.END_OF_FEED; +import static com.yahoo.document.json.document.DocumentParser.FIELDS; import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart; /** @@ -60,7 +61,7 @@ public class JsonReader { * @param docIdString document ID * @return the parsed document operation */ - public ParsedDocumentOperation readSingleDocument(DocumentOperationType operationType, String docIdString) { + ParsedDocumentOperation readSingleDocument(DocumentOperationType operationType, String docIdString) { DocumentId docId = new DocumentId(docIdString); DocumentParseInfo documentParseInfo; try { @@ -78,6 +79,54 @@ public class JsonReader { return operation; } + /** + * Reads a JSON which is expected to contain only the "fields" object of a document, + * and where other parameters, like the document ID and operation type, are supplied by other means. + * + * @param operationType the type of operation (update or put) + * @param docIdString document ID + * @return the parsed document operation + */ + public ParsedDocumentOperation readSingleDocumentStreaming(DocumentOperationType operationType, String docIdString) { + try { + DocumentId docId = new DocumentId(docIdString); + DocumentParseInfo documentParseInfo = new DocumentParseInfo(); + documentParseInfo.documentId = docId; + documentParseInfo.operationType = operationType; + + if (JsonToken.START_OBJECT != parser.nextValue()) + throw new IllegalArgumentException("expected start of root object, got " + parser.currentToken()); + + parser.nextValue(); + if ( ! FIELDS.equals(parser.getCurrentName())) + throw new IllegalArgumentException("expected field \"fields\", but got " + parser.getCurrentName()); + + if (JsonToken.START_OBJECT != parser.currentToken()) + throw new IllegalArgumentException("expected start of \"fields\" object, got " + parser.currentToken()); + + documentParseInfo.fieldsBuffer = new LazyTokenBuffer(parser); + VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(typeManager.getIgnoreUndefinedFields()); + ParsedDocumentOperation operation = vespaJsonDocumentReader.createDocumentOperation( + getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager), documentParseInfo); + + if ( ! documentParseInfo.fieldsBuffer.isEmpty()) + throw new IllegalArgumentException("expected all content to be consumed by document parsing, but " + + documentParseInfo.fieldsBuffer.nesting() + " levels remain"); + + if (JsonToken.END_OBJECT != parser.currentToken()) + throw new IllegalArgumentException("expected end of \"fields\" object, got " + parser.currentToken()); + if (JsonToken.END_OBJECT != parser.nextToken()) + throw new IllegalArgumentException("expected end of root object, got " + parser.currentToken()); + if (null != parser.nextToken()) + throw new IllegalArgumentException("expected end of input, got " + parser.currentToken()); + + return operation; + } + catch (IOException e) { + throw new IllegalArgumentException("failed parsing document", e); + } + } + /** Returns the next document operation, or null if we have reached the end */ public DocumentOperation next() { switch (state) { diff --git a/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java b/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java new file mode 100644 index 00000000000..0fbdd0b28c7 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java @@ -0,0 +1,64 @@ +package com.yahoo.document.json; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + +import java.io.IOException; +import java.util.function.Supplier; + +/** + * A {@link TokenBuffer} which only buffers tokens when needed, i.e., when peeking. + * + * @author jonmv + */ +public class LazyTokenBuffer extends TokenBuffer { + + private final JsonParser parser; + + public LazyTokenBuffer(JsonParser parser) { + this.parser = parser; + try { addFromParser(parser); } + catch (IOException e) { throw new IllegalArgumentException("failed parsing document JSON", e); } + if (JsonToken.START_OBJECT != current()) + throw new IllegalArgumentException("expected start of JSON object, but got " + current()); + updateNesting(current()); + } + + void advance() { + super.advance(); + if (tokens.isEmpty() && nesting() > 0) tokens.add(nextToken()); // Fill current token if needed and possible. + } + + @Override + public Supplier lookahead() { + return new Supplier<>() { + int localNesting = nesting(); + Supplier buffered = LazyTokenBuffer.super.lookahead(); + @Override public Token get() { + if (localNesting == 0) + return null; + + Token token = buffered.get(); + if (token == null) { + token = nextToken(); + tokens.add(token); + } + localNesting += nestingOffset(token.token); + return token; + } + }; + } + + private Token nextToken() { + try { + JsonToken token = parser.nextValue(); + if (token == null) + throw new IllegalStateException("no more JSON tokens"); + return new Token(token, parser.getCurrentName(), parser.getText()); + } + catch (IOException e) { + throw new IllegalArgumentException("failed reading document JSON", e); + } + } + +} diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java index dec84e46b77..3a48f71c4cd 100644 --- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java +++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java @@ -1,15 +1,16 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.json; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; import com.google.common.base.Preconditions; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.function.Supplier; + /** * Helper class to enable lookahead in the token stream. * @@ -17,101 +18,76 @@ import com.google.common.base.Preconditions; */ public class TokenBuffer { - private final List tokens; + final Deque tokens = new ArrayDeque<>(); - private int position = 0; private int nesting = 0; - public TokenBuffer() { - this(new ArrayList<>()); - } - - public TokenBuffer(List tokens) { - this.tokens = tokens; - if (tokens.size() > 0) - updateNesting(tokens.get(position).token); - } + public TokenBuffer() { } /** Returns whether any tokens are available in this */ - public boolean isEmpty() { return remaining() == 0; } - - public JsonToken previous() { - updateNestingGoingBackwards(current()); - position--; - return current(); - } - - /** Returns the current token without changing position, or null if none */ - public JsonToken current() { - if (isEmpty()) return null; - Token token = tokens.get(position); - if (token == null) return null; - return token.token; - } + public boolean isEmpty() { return tokens.isEmpty(); } + /** Returns the next token, or null, and updates the nesting count of this. */ public JsonToken next() { - position++; + advance(); JsonToken token = current(); updateNesting(token); return token; } - /** Returns a given number of tokens ahead, or null if none */ - public JsonToken peek(int ahead) { - if (tokens.size() <= position + ahead) return null; - return tokens.get(position + ahead).token; + void advance() { + tokens.poll(); + } + + /** Returns the current token without changing position, or null if none */ + public JsonToken current() { + return isEmpty() ? null : tokens.peek().token; } /** Returns the current token name without changing position, or null if none */ public String currentName() { - if (isEmpty()) return null; - Token token = tokens.get(position); - if (token == null) return null; - return token.name; + return isEmpty() ? null : tokens.peek().name; } /** Returns the current token text without changing position, or null if none */ public String currentText() { - if (isEmpty()) return null; - Token token = tokens.get(position); - if (token == null) return null; - return token.text; + return isEmpty() ? null : tokens.peek().text; } - public int remaining() { - return tokens.size() - position; + /** + * Returns a sequence of remaining tokens in this, or nulls when none remain. + * This may fill the token buffer, but not otherwise modify it. + */ + public Supplier lookahead() { + Iterator iterator = tokens.iterator(); + if (iterator.hasNext()) iterator.next(); + return () -> iterator.hasNext() ? iterator.next() : null; } private void add(JsonToken token, String name, String text) { - tokens.add(tokens.size(), new Token(token, name, text)); + tokens.add(new Token(token, name, text)); } - public void bufferObject(JsonToken first, JsonParser tokens) { - bufferJsonStruct(first, tokens, JsonToken.START_OBJECT); + public void bufferObject(JsonParser parser) { + bufferJsonStruct(parser, JsonToken.START_OBJECT); } - private void bufferJsonStruct(JsonToken first, JsonParser tokens, JsonToken firstToken) { - int localNesting = 0; - JsonToken t = first; + private void bufferJsonStruct(JsonParser parser, JsonToken firstToken) { + JsonToken token = parser.currentToken(); + Preconditions.checkArgument(token == firstToken, + "Expected %s, got %s.", firstToken.name(), token); + updateNesting(token); - Preconditions.checkArgument(first == firstToken, - "Expected %s, got %s.", firstToken.name(), t); - if (remaining() == 0) { - updateNesting(t); + try { + for (int nesting = addFromParser(parser); nesting > 0; nesting += addFromParser(parser)) + parser.nextValue(); } - localNesting = storeAndPeekNesting(t, localNesting, tokens); - while (localNesting > 0) { - t = nextValue(tokens); - localNesting = storeAndPeekNesting(t, localNesting, tokens); + catch (IOException e) { + throw new IllegalArgumentException(e); } } - private int storeAndPeekNesting(JsonToken t, int nesting, JsonParser tokens) { - addFromParser(t, tokens); - return nesting + nestingOffset(t); - } - - private int nestingOffset(JsonToken token) { + int nestingOffset(JsonToken token) { if (token == null) return 0; if (token.isStructStart()) { return 1; @@ -122,43 +98,23 @@ public class TokenBuffer { } } - private void addFromParser(JsonToken t, JsonParser tokens) { - try { - add(t, tokens.getCurrentName(), tokens.getText()); - } catch (IOException e) { - throw new IllegalArgumentException(e); - } - } - - private JsonToken nextValue(JsonParser tokens) { - try { - return tokens.nextValue(); - } catch (IOException e) { - throw new IllegalArgumentException(e); - } + int addFromParser(JsonParser tokens) throws IOException { + add(tokens.currentToken(), tokens.getCurrentName(), tokens.getText()); + return nestingOffset(tokens.currentToken()); } - private void updateNesting(JsonToken token) { + void updateNesting(JsonToken token) { nesting += nestingOffset(token); } - private void updateNestingGoingBackwards(JsonToken token) { - nesting -= nestingOffset(token); - } - public int nesting() { return nesting; } public void skipToRelativeNesting(int relativeNesting) { int initialNesting = nesting(); - do { - next(); - } while ( nesting() > initialNesting + relativeNesting); - } - - public List rest() { - return tokens.subList(position, tokens.size()); + do next(); + while (nesting() > initialNesting + relativeNesting); } public static final class Token { diff --git a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java index 74656762fe1..aef7e1cffe2 100644 --- a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java +++ b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java @@ -86,16 +86,6 @@ public class DocumentParser { private void handleIdentLevelOne(DocumentParseInfo documentParseInfo, boolean docIdAndOperationIsSetExternally) throws IOException { JsonToken currentToken = parser.getCurrentToken(); - if (currentToken == JsonToken.VALUE_TRUE || currentToken == JsonToken.VALUE_FALSE) { - try { - if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { - documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue()); - return; - } - } catch (IOException e) { - throw new RuntimeException("Got IO exception while parsing document", e); - } - } if ((currentToken == JsonToken.VALUE_TRUE || currentToken == JsonToken.VALUE_FALSE) && CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { documentParseInfo.create = Optional.of(currentToken == JsonToken.VALUE_TRUE); @@ -111,12 +101,11 @@ public class DocumentParser { } } - private void handleIdentLevelTwo(DocumentParseInfo documentParseInfo) { + private void handleIdentLevelTwo(DocumentParseInfo documentParseInfo) { try { - JsonToken currentToken = parser.getCurrentToken(); // "fields" opens a dictionary and is therefore on level two which might be surprising. - if (currentToken == JsonToken.START_OBJECT && FIELDS.equals(parser.getCurrentName())) { - documentParseInfo.fieldsBuffer.bufferObject(currentToken, parser); + if (parser.currentToken() == JsonToken.START_OBJECT && FIELDS.equals(parser.getCurrentName())) { + documentParseInfo.fieldsBuffer.bufferObject(parser); processIndent(); } } catch (IOException e) { diff --git a/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java index 2dce07cdbe6..e859306f04d 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java +++ b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java @@ -8,6 +8,7 @@ import com.yahoo.document.json.TokenBuffer; import java.util.Optional; public class DocumentParseInfo { + public DocumentParseInfo() { } public DocumentId documentId; public Optional create = Optional.empty(); public Optional condition = Optional.empty(); diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java index 0b7b1ae9996..1fd4029b1a5 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java @@ -4,13 +4,15 @@ package com.yahoo.document.json.readers; import com.fasterxml.jackson.core.JsonToken; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.json.TokenBuffer; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Type; +import com.yahoo.document.json.TokenBuffer.Token; import com.yahoo.tensor.IndexedTensor; import com.yahoo.tensor.MixedTensor; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.TensorType.Dimension; + +import java.util.function.Supplier; import static com.yahoo.document.json.readers.JsonParserHelpers.*; import static com.yahoo.tensor.serialization.JsonFormat.decodeHexString; @@ -37,36 +39,43 @@ public class TensorReader { Tensor.Builder builder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType()); expectOneOf(buffer.current(), JsonToken.START_OBJECT, JsonToken.START_ARRAY); int initNesting = buffer.nesting(); - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { - if (TENSOR_CELLS.equals(buffer.currentName()) && ! primitiveContent(buffer)) { + while (true) { + Supplier lookahead = buffer.lookahead(); + Token next = lookahead.get(); + if (TENSOR_CELLS.equals(next.name) && ! primitiveContent(next.token, lookahead.get().token)) { + buffer.next(); readTensorCells(buffer, builder); } - else if (TENSOR_VALUES.equals(buffer.currentName()) && builder.type().dimensions().stream().allMatch(d -> d.isIndexed())) { + else if (TENSOR_VALUES.equals(next.name) && builder.type().dimensions().stream().allMatch(Dimension::isIndexed)) { + buffer.next(); readTensorValues(buffer, builder); } - else if (TENSOR_BLOCKS.equals(buffer.currentName())) { + else if (TENSOR_BLOCKS.equals(next.name)) { + buffer.next(); readTensorBlocks(buffer, builder); } - else if (TENSOR_TYPE.equals(buffer.currentName()) && buffer.current() == JsonToken.VALUE_STRING) { + else if (TENSOR_TYPE.equals(next.name) && next.token == JsonToken.VALUE_STRING) { + buffer.next(); // Ignore input tensor type } + else if (buffer.nesting() == initNesting && JsonToken.END_OBJECT == next.token) { + buffer.next(); + break; + } else { - buffer.previous(); // Back up to the start of the enclosing block readDirectTensorValue(buffer, builder); - buffer.previous(); // ... and back up to the end of the enclosing block + break; } } expectOneOf(buffer.current(), JsonToken.END_OBJECT, JsonToken.END_ARRAY); tensorFieldValue.assign(builder.build()); } - static boolean primitiveContent(TokenBuffer buffer) { - JsonToken cellsValue = buffer.current(); - if (cellsValue.isScalarValue()) return true; - if (cellsValue == JsonToken.START_ARRAY) { - JsonToken firstArrayValue = buffer.peek(1); - if (firstArrayValue == JsonToken.END_ARRAY) return false; - if (firstArrayValue.isScalarValue()) return true; + static boolean primitiveContent(JsonToken current, JsonToken next) { + if (current.isScalarValue()) return true; + if (current == JsonToken.START_ARRAY) { + if (next == JsonToken.END_ARRAY) return false; + if (next.isScalarValue()) return true; } return false; } @@ -186,7 +195,7 @@ public class TensorReader { boolean hasIndexed = builder.type().dimensions().stream().anyMatch(TensorType.Dimension::isIndexed); boolean hasMapped = builder.type().dimensions().stream().anyMatch(TensorType.Dimension::isMapped); - if (isArrayOfObjects(buffer, 0)) + if (isArrayOfObjects(buffer)) readTensorCells(buffer, builder); else if ( ! hasMapped) readTensorValues(buffer, builder); @@ -196,10 +205,12 @@ public class TensorReader { readTensorCells(buffer, builder); } - private static boolean isArrayOfObjects(TokenBuffer buffer, int ahead) { - if (buffer.peek(ahead++) != JsonToken.START_ARRAY) return false; - if (buffer.peek(ahead) == JsonToken.START_ARRAY) return isArrayOfObjects(buffer, ahead); // nested array - return buffer.peek(ahead) == JsonToken.START_OBJECT; + private static boolean isArrayOfObjects(TokenBuffer buffer) { + if (buffer.current() != JsonToken.START_ARRAY) return false; + Supplier lookahead = buffer.lookahead(); + Token next; + while ((next = lookahead.get()).token == JsonToken.START_ARRAY) { } + return next.token == JsonToken.START_OBJECT; } private static TensorAddress readAddress(TokenBuffer buffer, TensorType type) { diff --git a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java index 113b8732b23..c7303d31ea2 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java @@ -238,7 +238,7 @@ public class VespaJsonDocumentReader { "Expected end of JSON struct (%s), got %s", expectedFinalToken, buffer.current()); Preconditions.checkState(buffer.nesting() == 0, "Nesting not zero at end of operation"); Preconditions.checkState(buffer.next() == null, "Dangling data at end of operation"); - Preconditions.checkState(buffer.remaining() == 0, "Dangling data at end of operation"); + Preconditions.checkState(buffer.isEmpty(), "Dangling data at end of operation"); } } diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 5a9f02c790d..080528fea77 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -2120,69 +2120,93 @@ public class JsonReaderTestCase { @Test public void tensor_modify_update_with_replace_operation() { assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.REPLACE, "sparse_tensor", - inputJson("{", - " 'operation': 'replace',", - " 'cells': [", - " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}")); + """ + { + "operation": "replace", + "cells": [ + { "address": { "x": "a", "y": "b" }, "value": 2.0 } + ] + }"""); } @Test public void tensor_modify_update_with_add_operation() { assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.ADD, "sparse_tensor", - inputJson("{", - " 'operation': 'add',", - " 'cells': [", - " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}")); + """ + { + "operation": "add", + "cells": [ + { "address": { "x": "a", "y": "b" }, "value": 2.0 } + ] + }"""); } @Test public void tensor_modify_update_with_multiply_operation() { assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.MULTIPLY, "sparse_tensor", - inputJson("{", - " 'operation': 'multiply',", - " 'cells': [", - " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}")); + """ + { + "operation": "multiply", + "cells": [ + { "address": { "x": "a", "y": "b" }, "value": 2.0 } + ] + }"""); } @Test public void tensor_modify_update_with_create_non_existing_cells_true() { assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.ADD, true, "sparse_tensor", - inputJson("{", - " 'operation': 'add',", - " 'create': true,", - " 'cells': [", - " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}")); + """ + { + "operation": "add", + "create": true, + "cells": [ + { "address": { "x": "a", "y": "b" }, "value": 2.0 } + ] + }"""); } @Test public void tensor_modify_update_with_create_non_existing_cells_false() { assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.ADD, false, "sparse_tensor", - inputJson("{", - " 'operation': 'add',", - " 'create': false,", - " 'cells': [", - " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}")); + """ + { + "operation": "add", + "create": false, + "cells": [ + { "address": { "x": "a", "y": "b" }, "value": 2.0 } + ] + }"""); } @Test public void tensor_modify_update_treats_the_input_tensor_as_sparse() { // Note that the type of the tensor in the modify update is sparse (it only has mapped dimensions). assertTensorModifyUpdate("tensor(x{},y{}):{{x:0,y:0}:2.0, {x:1,y:2}:3.0}", - TensorModifyUpdate.Operation.REPLACE, "dense_tensor", - inputJson("{", - " 'operation': 'replace',", - " 'cells': [", - " { 'address': { 'x': '0', 'y': '0' }, 'value': 2.0 },", - " { 'address': { 'x': '1', 'y': '2' }, 'value': 3.0 } ]}")); + TensorModifyUpdate.Operation.REPLACE, "dense_tensor", + """ + { + "operation": "replace", + "cells": [ + { "address": { "x": "0", "y": "0" }, "value": 2.0 }, + { "address": { "x": "1", "y": "2" }, "value": 3.0 } + ] + }"""); } @Test public void tensor_modify_update_on_non_tensor_field_throws() { try { - JsonReader reader = createReader(inputJson("{ 'update': 'id:unittest:smoke::doc1',", - " 'fields': {", - " 'something': {", - " 'modify': {} }}}")); + JsonReader reader = createReader(""" + { + "update": "id:unittest:smoke::doc1", + "fields": { + "something": { + "modify": {} + } + } + } + """); reader.readSingleDocument(DocumentOperationType.UPDATE, "id:unittest:smoke::doc1"); fail("Expected exception"); } @@ -2196,95 +2220,125 @@ public class JsonReaderTestCase { public void tensor_modify_update_on_dense_unbound_tensor_throws() { illegalTensorModifyUpdate("Error in 'dense_unbound_tensor': A modify update cannot be applied to tensor types with indexed unbound dimensions. Field 'dense_unbound_tensor' has unsupported tensor type 'tensor(x[],y[])'", "dense_unbound_tensor", - "{", - " 'operation': 'replace',", - " 'cells': [", - " { 'address': { 'x': '0', 'y': '0' }, 'value': 2.0 } ]}"); + """ + { + "operation": "replace", + "cells": [ + { "address": { "x": "0", "y": "0" }, "value": 2.0 } + ] + }"""); } @Test public void tensor_modify_update_on_sparse_tensor_with_single_dimension_short_form() { - assertTensorModifyUpdate("{{x:a}:2.0, {x:c}: 3.0}", TensorModifyUpdate.Operation.REPLACE, "sparse_single_dimension_tensor", - inputJson("{", - " 'operation': 'replace',", - " 'cells': {", - " 'a': 2.0,", - " 'c': 3.0 }}")); + assertTensorModifyUpdate("{{x:a}:2.0, {x:c}: 3.0}", TensorModifyUpdate.Operation.REPLACE, "sparse_single_dimension_tensor", + """ + { + "operation": "replace", + "cells": { + "a": 2.0, + "c": 3.0 + } + }"""); } @Test public void tensor_modify_update_with_replace_operation_mixed() { assertTensorModifyUpdate("{{x:a,y:0}:2.0}", TensorModifyUpdate.Operation.REPLACE, "mixed_tensor", - inputJson("{", - " 'operation': 'replace',", - " 'cells': [", - " { 'address': { 'x': 'a', 'y': '0' }, 'value': 2.0 } ]}")); + """ + { + "operation": "replace", + "cells": [ + { "address": { "x": "a", "y": "0" }, "value": 2.0 } + ] + }"""); } @Test public void tensor_modify_update_with_replace_operation_mixed_block_short_form_array() { assertTensorModifyUpdate("{{x:a,y:0}:1,{x:a,y:1}:2,{x:a,y:2}:3}", TensorModifyUpdate.Operation.REPLACE, "mixed_tensor", - inputJson("{", - " 'operation': 'replace',", - " 'blocks': [", - " { 'address': { 'x': 'a' }, 'values': [1,2,3] } ]}")); + """ + { + "operation": "replace", + "blocks": [ + { "address": { "x": "a" }, "values": [1,2,3] } + ] + }"""); } @Test public void tensor_modify_update_with_replace_operation_mixed_block_short_form_must_specify_full_subspace() { illegalTensorModifyUpdate("Error in 'mixed_tensor': At {x:a}: Expected 3 values, but got 2", - "mixed_tensor", - inputJson("{", - " 'operation': 'replace',", - " 'blocks': {", - " 'a': [2,3] } }")); + "mixed_tensor", + """ + { + "operation": "replace", + "blocks": { + "a": [2,3] + } + }"""); } @Test public void tensor_modify_update_with_replace_operation_mixed_block_short_form_map() { assertTensorModifyUpdate("{{x:a,y:0}:1,{x:a,y:1}:2,{x:a,y:2}:3}", TensorModifyUpdate.Operation.REPLACE, "mixed_tensor", - inputJson("{", - " 'operation': 'replace',", - " 'blocks': {", - " 'a': [1,2,3] } }")); + """ + { + "operation": "replace", + "blocks": { + "a": [1,2,3] + } + }"""); } @Test public void tensor_modify_update_with_add_operation_mixed() { assertTensorModifyUpdate("{{x:a,y:0}:2.0}", TensorModifyUpdate.Operation.ADD, "mixed_tensor", - inputJson("{", - " 'operation': 'add',", - " 'cells': [", - " { 'address': { 'x': 'a', 'y': '0' }, 'value': 2.0 } ]}")); + """ + { + "operation": "add", + "cells": [ + { "address": { "x": "a", "y": "0" }, "value": 2.0 } + ] + }"""); } @Test public void tensor_modify_update_with_multiply_operation_mixed() { assertTensorModifyUpdate("{{x:a,y:0}:2.0}", TensorModifyUpdate.Operation.MULTIPLY, "mixed_tensor", - inputJson("{", - " 'operation': 'multiply',", - " 'cells': [", - " { 'address': { 'x': 'a', 'y': '0' }, 'value': 2.0 } ]}")); + """ + { + "operation": "multiply", + "cells": [ + { "address": { "x": "a", "y": "0" }, "value": 2.0 } + ] + }"""); } @Test public void tensor_modify_update_with_out_of_bound_cells_throws() { illegalTensorModifyUpdate("Error in 'dense_tensor': Dimension 'y' has label '3' but type is tensor(x[2],y[3])", "dense_tensor", - "{", - " 'operation': 'replace',", - " 'cells': [", - " { 'address': { 'x': '0', 'y': '3' }, 'value': 2.0 } ]}"); + """ + { + "operation": "replace", + "cells": [ + { "address": { "x": "0", "y": "3" }, "value": 2.0 } + ] + }"""); } @Test public void tensor_modify_update_with_out_of_bound_cells_throws_mixed() { illegalTensorModifyUpdate("Error in 'mixed_tensor': Dimension 'y' has label '3' but type is tensor(x{},y[3])", "mixed_tensor", - "{", - " 'operation': 'replace',", - " 'cells': [", - " { 'address': { 'x': '0', 'y': '3' }, 'value': 2.0 } ]}"); + """ + { + "operation": "replace", + "cells": [ + { "address": { "x": "0", "y": "3" }, "value": 2.0 } + ] + }"""); } @@ -2292,87 +2346,113 @@ public class JsonReaderTestCase { public void tensor_modify_update_with_unknown_operation_throws() { illegalTensorModifyUpdate("Error in 'sparse_tensor': Unknown operation 'unknown' in modify update for field 'sparse_tensor'", "sparse_tensor", - "{", - " 'operation': 'unknown',", - " 'cells': [", - " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}"); + """ + { + "operation": "unknown", + "cells": [ + { "address": { "x": "a", "y": "b" }, "value": 2.0 } + ] + }"""); } @Test public void tensor_modify_update_without_operation_throws() { illegalTensorModifyUpdate("Error in 'sparse_tensor': Modify update for field 'sparse_tensor' does not contain an operation", "sparse_tensor", - "{", - " 'cells': [] }"); + """ + { + "cells": [] + }"""); } @Test public void tensor_modify_update_without_cells_throws() { illegalTensorModifyUpdate("Error in 'sparse_tensor': Modify update for field 'sparse_tensor' does not contain tensor cells", "sparse_tensor", - "{", - " 'operation': 'replace' }"); + """ + { + "operation": "replace" + }"""); } @Test public void tensor_modify_update_with_unknown_content_throws() { illegalTensorModifyUpdate("Error in 'sparse_tensor': Unknown JSON string 'unknown' in modify update for field 'sparse_tensor'", "sparse_tensor", - "{", - " 'unknown': 'here' }"); + """ + { + "unknown": "here" + }"""); } @Test public void tensor_add_update_on_sparse_tensor() { assertTensorAddUpdate("{{x:a,y:b}:2.0, {x:c,y:d}: 3.0}", "sparse_tensor", - inputJson("{", - " 'cells': [", - " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 },", - " { 'address': { 'x': 'c', 'y': 'd' }, 'value': 3.0 } ]}")); + """ + { + "cells": [ + { "address": { "x": "a", "y": "b" }, "value": 2.0 }, + { "address": { "x": "c", "y": "d" }, "value": 3.0 } + ] + }"""); } @Test public void tensor_add_update_on_sparse_tensor_with_single_dimension_short_form() { assertTensorAddUpdate("{{x:a}:2.0, {x:c}: 3.0}", "sparse_single_dimension_tensor", - inputJson("{", - " 'cells': {", - " 'a': 2.0,", - " 'c': 3.0 }}")); + """ + { + "cells": { + "a": 2.0, + "c": 3.0 + } + }"""); } @Test public void tensor_add_update_on_mixed_tensor() { assertTensorAddUpdate("{{x:a,y:0}:2.0, {x:a,y:1}:3.0, {x:a,y:2}:0.0}", "mixed_tensor", - inputJson("{", - " 'cells': [", - " { 'address': { 'x': 'a', 'y': '0' }, 'value': 2.0 },", - " { 'address': { 'x': 'a', 'y': '1' }, 'value': 3.0 } ]}")); + """ + { + "cells": [ + { "address": { "x": "a", "y": "0" }, "value": 2.0 }, + { "address": { "x": "a", "y": "1" }, "value": 3.0 } + ] + }"""); } @Test public void tensor_add_update_on_mixed_with_out_of_bound_dense_cells_throws() { illegalTensorAddUpdate("Error in 'mixed_tensor': Index 3 out of bounds for length 3", "mixed_tensor", - "{", - " 'cells': [", - " { 'address': { 'x': '0', 'y': '3' }, 'value': 2.0 } ]}"); + """ + { + "cells": [ + { "address": { "x": "0", "y": "3" }, "value": 2.0 } + ] + }"""); } @Test public void tensor_add_update_on_dense_tensor_throws() { illegalTensorAddUpdate("Error in 'dense_tensor': An add update can only be applied to tensors with at least one sparse dimension. Field 'dense_tensor' has unsupported tensor type 'tensor(x[2],y[3])'", "dense_tensor", - "{", - " 'cells': [] }"); + """ + { + "cells": [ ] + }"""); } @Test public void tensor_add_update_on_not_fully_specified_cell_throws() { illegalTensorAddUpdate("Error in 'sparse_tensor': Missing a label for dimension 'y' for tensor(x{},y{})", "sparse_tensor", - "{", - " 'cells': [", - " { 'address': { 'x': 'a' }, 'value': 2.0 } ]}"); + """ + { + "cells": [ + { "address": { "x": "a" }, "value": 2.0 } + ] + }"""); } @Test @@ -2388,146 +2468,176 @@ public class JsonReaderTestCase { @Test public void tensor_remove_update_on_sparse_tensor() { assertTensorRemoveUpdate("{{x:a,y:b}:1.0,{x:c,y:d}:1.0}", "sparse_tensor", - inputJson("{", - " 'addresses': [", - " { 'x': 'a', 'y': 'b' },", - " { 'x': 'c', 'y': 'd' } ]}")); + """ + { + "addresses": [ + { "x": "a", "y": "b" }, + { "x": "c", "y": "d" } + ] + }"""); } @Test public void tensor_remove_update_on_mixed_tensor() { assertTensorRemoveUpdate("{{x:1}:1.0,{x:2}:1.0}", "mixed_tensor", - inputJson("{", - " 'addresses': [", - " { 'x': '1' },", - " { 'x': '2' } ]}")); + """ + { + "addresses": [ + { "x": "1" }, + { "x": "2" } + ] + }"""); } @Test public void tensor_remove_update_on_sparse_tensor_with_not_fully_specified_address() { assertTensorRemoveUpdate("{{y:b}:1.0,{y:d}:1.0}", "sparse_tensor", - inputJson("{", - " 'addresses': [", - " { 'y': 'b' },", - " { 'y': 'd' } ]}")); + """ + { + "addresses": [ + { "y": "b" }, + { "y": "d" } + ] + }"""); } @Test public void tensor_remove_update_on_mixed_tensor_with_not_fully_specified_address() { assertTensorRemoveUpdate("{{x:1,z:a}:1.0,{x:2,z:b}:1.0}", "mixed_tensor_adv", - inputJson("{", - " 'addresses': [", - " { 'x': '1', 'z': 'a' },", - " { 'x': '2', 'z': 'b' } ]}")); + """ + { + "addresses": [ + { "x": "1", "z": "a" }, + { "x": "2", "z": "b" } + ] + }"""); } @Test public void tensor_remove_update_on_mixed_tensor_with_dense_addresses_throws() { illegalTensorRemoveUpdate("Error in 'mixed_tensor': Indexed dimension address 'y' should not be specified in remove update", "mixed_tensor", - "{", - " 'addresses': [", - " { 'x': '1', 'y': '0' },", - " { 'x': '2', 'y': '0' } ]}"); + """ + { + "addresses": [ + { "x": "1", "y": "0" }, + { "x": "2", "y": "0" } + ] + }"""); } @Test public void tensor_remove_update_on_dense_tensor_throws() { illegalTensorRemoveUpdate("Error in 'dense_tensor': A remove update can only be applied to tensors with at least one sparse dimension. Field 'dense_tensor' has unsupported tensor type 'tensor(x[2],y[3])'", "dense_tensor", - "{", - " 'addresses': [] }"); + """ + { + "addresses": [] + }"""); } @Test public void tensor_remove_update_with_stray_dimension_throws() { illegalTensorRemoveUpdate("Error in 'sparse_tensor': tensor(x{},y{}) does not contain dimension 'foo'", - "sparse_tensor", - "{", - " 'addresses': [", - " { 'x': 'a', 'foo': 'b' } ]}"); + "sparse_tensor", + """ + { + "addresses": [ + { "x": "a", "foo": "b" } + ] + }"""); illegalTensorRemoveUpdate("Error in 'sparse_tensor': tensor(x{}) does not contain dimension 'foo'", - "sparse_tensor", - "{", - " 'addresses': [", - " { 'x': 'c' },", - " { 'x': 'a', 'foo': 'b' } ]}"); + "sparse_tensor", + """ + { + "addresses": [ + { "x": "c" }, + { "x": "a", "foo": "b" } + ] + }"""); } @Test public void tensor_remove_update_without_cells_throws() { illegalTensorRemoveUpdate("Error in 'sparse_tensor': Remove update for field 'sparse_tensor' does not contain tensor addresses", "sparse_tensor", - "{'addresses': [] }"); + """ + { + "addresses": [] + }"""); illegalTensorRemoveUpdate("Error in 'mixed_tensor': Remove update for field 'mixed_tensor' does not contain tensor addresses", "mixed_tensor", - "{'addresses': [] }"); + """ + { + "addresses": [] + }"""); } @Test public void require_that_parser_propagates_datatype_parser_errors_predicate() { assertParserErrorMatches( "Error in document 'id:unittest:testpredicate::0' - could not parse field 'boolean' of type 'predicate': " + - "line 1:10 no viable alternative at character '>'", - - "[", - " {", - " 'fields': {", - " 'boolean': 'timestamp > 9000'", - " },", - " 'put': 'id:unittest:testpredicate::0'", - " }", - "]" - ); + "line 1:10 no viable alternative at character '>'", + """ + [ + { + "fields": { + "boolean": "timestamp > 9000" + }, + "put": "id:unittest:testpredicate::0" + } + ] + """); } @Test public void require_that_parser_propagates_datatype_parser_errors_string_as_int() { assertParserErrorMatches( "Error in document 'id:unittest:testint::0' - could not parse field 'integerfield' of type 'int': " + - "For input string: \" 1\"", - - "[", - " {", - " 'fields': {", - " 'integerfield': ' 1'", - " },", - " 'put': 'id:unittest:testint::0'", - " }", - "]" - ); + "For input string: \" 1\"", + """ + [ + { + "fields": { + "integerfield": " 1" + }, + "put": "id:unittest:testint::0" + } + ] + """); } @Test public void require_that_parser_propagates_datatype_parser_errors_overflowing_int() { assertParserErrorMatches( "Error in document 'id:unittest:testint::0' - could not parse field 'integerfield' of type 'int': " + - "For input string: \"281474976710656\"", - - "[", - " {", - " 'fields': {", - " 'integerfield': 281474976710656", - " },", - " 'put': 'id:unittest:testint::0'", - " }", - "]" - ); + "For input string: \"281474976710656\"", + """ + [ + { + "fields": { + "integerfield": 281474976710656 + }, + "put": "id:unittest:testint::0" + } + ] + """); } @Test public void requireThatUnknownDocTypeThrowsIllegalArgumentException() { - final String jsonData = inputJson( - "[", - " {", - " 'put': 'id:ns:walrus::walrus1',", - " 'fields': {", - " 'aField': 42", - " }", - " }", - "]"); + String jsonData = """ + [ + { + "put": "id:ns:walrus::walrus1", + "fields": { + "aField": 42 + } + } + ] + """; try { new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next(); fail(); @@ -2577,30 +2687,40 @@ public class JsonReaderTestCase { return createPutWithTensor(inputTensor, "sparse_tensor"); } private DocumentPut createPutWithTensor(String inputTensor, String tensorFieldName) { - JsonReader reader = createReader(inputJson("[", - "{ 'put': '" + TENSOR_DOC_ID + "',", - " 'fields': {", - " '" + tensorFieldName + "': " + inputTensor + " }}]")); - return (DocumentPut) reader.next(); + JsonReader streaming = createReader(""" + { + "fields": { + "%s": %s + } + } + """.formatted(tensorFieldName, inputTensor)); + DocumentPut lazyParsed = (DocumentPut) streaming.readSingleDocumentStreaming(DocumentOperationType.PUT, TENSOR_DOC_ID).operation(); + JsonReader reader = createReader(""" + [ + { + "put": "%s", + "fields": { + "%s": %s + } + } + ]""".formatted(TENSOR_DOC_ID, tensorFieldName, inputTensor)); + DocumentPut bufferParsed = (DocumentPut) reader.next(); + assertEquals(lazyParsed, bufferParsed); + return bufferParsed; } private DocumentUpdate createAssignUpdateWithSparseTensor(String inputTensor) { return createAssignUpdateWithTensor(inputTensor, "sparse_tensor"); } private DocumentUpdate createAssignUpdateWithTensor(String inputTensor, String tensorFieldName) { - JsonReader reader = createReader(inputJson("[", - "{ 'update': '" + TENSOR_DOC_ID + "',", - " 'fields': {", - " '" + tensorFieldName + "': {", - " 'assign': " + (inputTensor != null ? inputTensor : "null") + " } } } ]")); - return (DocumentUpdate) reader.next(); + return createTensorUpdate("assign", inputTensor, tensorFieldName); } private static Tensor assertSparseTensorField(String expectedTensor, DocumentPut put) { return assertTensorField(expectedTensor, put, "sparse_tensor"); } private Tensor assertTensorField(String expectedTensor, String fieldName, String inputJson) { - return assertTensorField(expectedTensor, createPutWithTensor(inputJson, fieldName), fieldName); + return assertTensorField(expectedTensor, createPutWithTensor(inputJson(inputJson), fieldName), fieldName); } private static Tensor assertTensorField(String expectedTensor, DocumentPut put, String tensorFieldName) { return assertTensorField(Tensor.from(expectedTensor), put, tensorFieldName); @@ -2673,12 +2793,29 @@ public class JsonReaderTestCase { } private DocumentUpdate createTensorUpdate(String operation, String tensorJson, String tensorFieldName) { - JsonReader reader = createReader(inputJson("[", - "{ 'update': '" + TENSOR_DOC_ID + "',", - " 'fields': {", - " '" + tensorFieldName + "': {", - " '" + operation + "': " + tensorJson + " }}}]")); - return (DocumentUpdate) reader.next(); + JsonReader streaming = createReader(""" + { + "fields": { + "%s": { + "%s": %s + } + } + }""".formatted(tensorFieldName, operation, tensorJson)); + DocumentUpdate lazyParsed = (DocumentUpdate) streaming.readSingleDocumentStreaming(DocumentOperationType.UPDATE, TENSOR_DOC_ID).operation(); + JsonReader reader = createReader(""" + [ + { + "update": "%s", + "fields": { + "%s": { + "%s": %s + } + } + } + ]""".formatted(TENSOR_DOC_ID, tensorFieldName, operation, tensorJson)); + DocumentUpdate bufferParsed = (DocumentUpdate) reader.next(); + assertEquals(lazyParsed, bufferParsed); + return bufferParsed; } private void assertTensorAddUpdate(String expectedTensor, String tensorFieldName, String tensorJson) { diff --git a/document/src/test/java/com/yahoo/document/json/LazyTokenBufferTest.java b/document/src/test/java/com/yahoo/document/json/LazyTokenBufferTest.java new file mode 100644 index 00000000000..3ed2ed531c3 --- /dev/null +++ b/document/src/test/java/com/yahoo/document/json/LazyTokenBufferTest.java @@ -0,0 +1,132 @@ +package com.yahoo.document.json; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.yahoo.document.json.TokenBuffer.Token; +import org.junit.Test; + +import java.io.IOException; +import java.util.function.Supplier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * @author jonmv + */ +public class LazyTokenBufferTest { + + @Test + public void testBuffer() throws IOException { + String json = """ + { + "fields": { + "foo": "bar", + "baz": [1, 2, 3], + "quu": { "qux": null } + } + }"""; + JsonParser parser = new JsonFactory().createParser(json); + parser.nextValue(); + parser.nextValue(); + assertEquals(JsonToken.START_OBJECT, parser.currentToken()); + assertEquals("fields", parser.currentName()); + + // Peeking through the buffer doesn't change nesting. + LazyTokenBuffer buffer = new LazyTokenBuffer(parser); + assertEquals(JsonToken.START_OBJECT, buffer.current()); + assertEquals("fields", buffer.currentName()); + assertEquals(1, buffer.nesting()); + + Supplier lookahead = buffer.lookahead(); + Token peek = lookahead.get(); + assertEquals(JsonToken.VALUE_STRING, peek.token); + assertEquals("foo", peek.name); + assertEquals("bar", peek.text); + assertEquals(1, buffer.nesting()); + + peek = lookahead.get(); + assertEquals(JsonToken.START_ARRAY, peek.token); + assertEquals("baz", peek.name); + assertEquals(1, buffer.nesting()); + + peek = lookahead.get(); + assertEquals(JsonToken.VALUE_NUMBER_INT, peek.token); + assertEquals("1", peek.text); + + peek = lookahead.get(); + assertEquals(JsonToken.VALUE_NUMBER_INT, peek.token); + assertEquals("2", peek.text); + + peek = lookahead.get(); + assertEquals(JsonToken.VALUE_NUMBER_INT, peek.token); + assertEquals("3", peek.text); + + peek = lookahead.get(); + assertEquals(JsonToken.END_ARRAY, peek.token); + assertEquals(1, buffer.nesting()); + + peek = lookahead.get(); + assertEquals(JsonToken.START_OBJECT, peek.token); + assertEquals("quu", peek.name); + assertEquals(1, buffer.nesting()); + + peek = lookahead.get(); + assertEquals(JsonToken.VALUE_NULL, peek.token); + assertEquals("qux", peek.name); + + peek = lookahead.get(); + assertEquals(JsonToken.END_OBJECT, peek.token); + assertEquals(1, buffer.nesting()); + + peek = lookahead.get(); + assertEquals(JsonToken.END_OBJECT, peek.token); + assertEquals(1, buffer.nesting()); + + peek = lookahead.get(); + assertNull(peek); + + // Parser is now at the end. + assertEquals(JsonToken.END_OBJECT, parser.nextToken()); + assertNull(parser.nextToken()); + + // Repeat iterating through the buffer, this time advancing it, and see that nesting changes. + assertEquals(JsonToken.VALUE_STRING, buffer.next()); + assertEquals("foo", buffer.currentName()); + assertEquals("bar", buffer.currentText()); + assertEquals(1, buffer.nesting()); + + assertEquals(JsonToken.START_ARRAY, buffer.next()); + assertEquals("baz", buffer.currentName()); + assertEquals(2, buffer.nesting()); + + assertEquals(JsonToken.VALUE_NUMBER_INT, buffer.next()); + assertEquals("1", buffer.currentText()); + + assertEquals(JsonToken.VALUE_NUMBER_INT, buffer.next()); + assertEquals("2", buffer.currentText()); + + assertEquals(JsonToken.VALUE_NUMBER_INT, buffer.next()); + assertEquals("3", buffer.currentText()); + + assertEquals(JsonToken.END_ARRAY, buffer.next()); + assertEquals(1, buffer.nesting()); + + assertEquals(JsonToken.START_OBJECT, buffer.next()); + assertEquals("quu", buffer.currentName()); + assertEquals(2, buffer.nesting()); + + assertEquals(JsonToken.VALUE_NULL, buffer.next()); + assertEquals("qux", buffer.currentName()); + + assertEquals(JsonToken.END_OBJECT, buffer.next()); + assertEquals(1, buffer.nesting()); + + assertEquals(JsonToken.END_OBJECT, buffer.next()); + assertEquals(0, buffer.nesting()); + + assertNull(buffer.next()); + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 5c63b07dcc0..5ff7b4592a1 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -1058,7 +1058,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private ParsedDocumentOperation parse(InputStream inputStream, String docId, DocumentOperationType operation) { try { - return new JsonReader(manager, inputStream, jsonFactory).readSingleDocument(operation, docId); + return new JsonReader(manager, inputStream, jsonFactory).readSingleDocumentStreaming(operation, docId); } catch (IllegalArgumentException e) { incrementMetricParseError(); throw e; diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index c8fcb4c4635..847b61b4af0 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -778,7 +778,7 @@ public class DocumentV1ApiTest { response = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two?condition=test%20it", POST, ""); assertSameJson("{" + " \"pathId\": \"/document/v1/space/music/number/1/two\"," + - " \"message\": \"Could not read document, no document?\"" + + " \"message\": \"expected start of root object, got null\"" + "}", response.readAll()); assertEquals(400, response.getStatus()); @@ -791,7 +791,8 @@ public class DocumentV1ApiTest { "}"); Inspector responseRoot = SlimeUtils.jsonToSlime(response.readAll()).get(); assertEquals("/document/v1/space/music/number/1/two", responseRoot.field("pathId").asString()); - assertTrue(responseRoot.field("message").asString().startsWith("Unexpected character ('┻' (code 9531 / 0x253b)): was expecting double-quote to start field name")); + assertTrue(responseRoot.field("message").asString(), + responseRoot.field("message").asString().startsWith("failed parsing document: Unexpected character ('┻' (code 9531 / 0x253b)): was expecting double-quote to start field name")); assertEquals(400, response.getStatus()); // PUT on a unknown document type is a 400 -- cgit v1.2.3