From 37ea99834dc8c4adf1dc03810dede2d0e766847d Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 25 Jan 2024 16:39:17 +0100 Subject: Revert "Jonmv/leaner token buffer" --- .../java/com/yahoo/document/json/JsonReader.java | 51 +- .../com/yahoo/document/json/LazyTokenBuffer.java | 64 --- .../java/com/yahoo/document/json/TokenBuffer.java | 140 ++++-- .../document/json/document/DocumentParser.java | 17 +- .../document/json/readers/DocumentParseInfo.java | 1 - .../yahoo/document/json/readers/TensorReader.java | 53 +- .../json/readers/VespaJsonDocumentReader.java | 2 +- .../yahoo/document/json/JsonReaderTestCase.java | 533 ++++++++------------- .../yahoo/document/json/LazyTokenBufferTest.java | 132 ----- 9 files changed, 327 insertions(+), 666 deletions(-) delete mode 100644 document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java delete mode 100644 document/src/test/java/com/yahoo/document/json/LazyTokenBufferTest.java (limited to 'document') diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index 08d1fe688ed..3e1743b8d45 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -18,7 +18,6 @@ import java.io.InputStream; import java.util.Optional; import static com.yahoo.document.json.JsonReader.ReaderState.END_OF_FEED; -import static com.yahoo.document.json.document.DocumentParser.FIELDS; import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart; /** @@ -61,7 +60,7 @@ public class JsonReader { * @param docIdString document ID * @return the parsed document operation */ - ParsedDocumentOperation readSingleDocument(DocumentOperationType operationType, String docIdString) { + public ParsedDocumentOperation readSingleDocument(DocumentOperationType operationType, String docIdString) { DocumentId docId = new DocumentId(docIdString); DocumentParseInfo documentParseInfo; try { @@ -79,54 +78,6 @@ public class JsonReader { return operation; } - /** - * Reads a JSON which is expected to contain only the "fields" object of a document, - * and where other parameters, like the document ID and operation type, are supplied by other means. - * - * @param operationType the type of operation (update or put) - * @param docIdString document ID - * @return the parsed document operation - */ - public ParsedDocumentOperation readSingleDocumentStreaming(DocumentOperationType operationType, String docIdString) { - try { - DocumentId docId = new DocumentId(docIdString); - DocumentParseInfo documentParseInfo = new DocumentParseInfo(); - documentParseInfo.documentId = docId; - documentParseInfo.operationType = operationType; - - if (JsonToken.START_OBJECT != parser.nextValue()) - throw new IllegalArgumentException("expected start of root object, got " + parser.currentToken()); - - parser.nextValue(); - if ( ! FIELDS.equals(parser.getCurrentName())) - throw new IllegalArgumentException("expected field \"fields\", but got " + parser.getCurrentName()); - - if (JsonToken.START_OBJECT != parser.currentToken()) - throw new IllegalArgumentException("expected start of \"fields\" object, got " + parser.currentToken()); - - documentParseInfo.fieldsBuffer = new LazyTokenBuffer(parser); - VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(typeManager.getIgnoreUndefinedFields()); - ParsedDocumentOperation operation = vespaJsonDocumentReader.createDocumentOperation( - getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager), documentParseInfo); - - if ( ! documentParseInfo.fieldsBuffer.isEmpty()) - throw new IllegalArgumentException("expected all content to be consumed by document parsing, but " + - documentParseInfo.fieldsBuffer.nesting() + " levels remain"); - - if (JsonToken.END_OBJECT != parser.currentToken()) - throw new IllegalArgumentException("expected end of \"fields\" object, got " + parser.currentToken()); - if (JsonToken.END_OBJECT != parser.nextToken()) - throw new IllegalArgumentException("expected end of root object, got " + parser.currentToken()); - if (null != parser.nextToken()) - throw new IllegalArgumentException("expected end of input, got " + parser.currentToken()); - - return operation; - } - catch (IOException e) { - throw new IllegalArgumentException("failed parsing document", e); - } - } - /** Returns the next document operation, or null if we have reached the end */ public DocumentOperation next() { switch (state) { diff --git a/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java b/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java deleted file mode 100644 index 0fbdd0b28c7..00000000000 --- a/document/src/main/java/com/yahoo/document/json/LazyTokenBuffer.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.yahoo.document.json; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; - -import java.io.IOException; -import java.util.function.Supplier; - -/** - * A {@link TokenBuffer} which only buffers tokens when needed, i.e., when peeking. - * - * @author jonmv - */ -public class LazyTokenBuffer extends TokenBuffer { - - private final JsonParser parser; - - public LazyTokenBuffer(JsonParser parser) { - this.parser = parser; - try { addFromParser(parser); } - catch (IOException e) { throw new IllegalArgumentException("failed parsing document JSON", e); } - if (JsonToken.START_OBJECT != current()) - throw new IllegalArgumentException("expected start of JSON object, but got " + current()); - updateNesting(current()); - } - - void advance() { - super.advance(); - if (tokens.isEmpty() && nesting() > 0) tokens.add(nextToken()); // Fill current token if needed and possible. - } - - @Override - public Supplier lookahead() { - return new Supplier<>() { - int localNesting = nesting(); - Supplier buffered = LazyTokenBuffer.super.lookahead(); - @Override public Token get() { - if (localNesting == 0) - return null; - - Token token = buffered.get(); - if (token == null) { - token = nextToken(); - tokens.add(token); - } - localNesting += nestingOffset(token.token); - return token; - } - }; - } - - private Token nextToken() { - try { - JsonToken token = parser.nextValue(); - if (token == null) - throw new IllegalStateException("no more JSON tokens"); - return new Token(token, parser.getCurrentName(), parser.getText()); - } - catch (IOException e) { - throw new IllegalArgumentException("failed reading document JSON", e); - } - } - -} diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java index 3a48f71c4cd..dec84e46b77 100644 --- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java +++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java @@ -1,16 +1,15 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.json; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; import com.google.common.base.Preconditions; -import java.io.IOException; -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.Iterator; -import java.util.function.Supplier; - /** * Helper class to enable lookahead in the token stream. * @@ -18,76 +17,101 @@ import java.util.function.Supplier; */ public class TokenBuffer { - final Deque tokens = new ArrayDeque<>(); + private final List tokens; + private int position = 0; private int nesting = 0; - public TokenBuffer() { } + public TokenBuffer() { + this(new ArrayList<>()); + } + + public TokenBuffer(List tokens) { + this.tokens = tokens; + if (tokens.size() > 0) + updateNesting(tokens.get(position).token); + } /** Returns whether any tokens are available in this */ - public boolean isEmpty() { return tokens.isEmpty(); } + public boolean isEmpty() { return remaining() == 0; } + + public JsonToken previous() { + updateNestingGoingBackwards(current()); + position--; + return current(); + } + + /** Returns the current token without changing position, or null if none */ + public JsonToken current() { + if (isEmpty()) return null; + Token token = tokens.get(position); + if (token == null) return null; + return token.token; + } - /** Returns the next token, or null, and updates the nesting count of this. */ public JsonToken next() { - advance(); + position++; JsonToken token = current(); updateNesting(token); return token; } - void advance() { - tokens.poll(); - } - - /** Returns the current token without changing position, or null if none */ - public JsonToken current() { - return isEmpty() ? null : tokens.peek().token; + /** Returns a given number of tokens ahead, or null if none */ + public JsonToken peek(int ahead) { + if (tokens.size() <= position + ahead) return null; + return tokens.get(position + ahead).token; } /** Returns the current token name without changing position, or null if none */ public String currentName() { - return isEmpty() ? null : tokens.peek().name; + if (isEmpty()) return null; + Token token = tokens.get(position); + if (token == null) return null; + return token.name; } /** Returns the current token text without changing position, or null if none */ public String currentText() { - return isEmpty() ? null : tokens.peek().text; + if (isEmpty()) return null; + Token token = tokens.get(position); + if (token == null) return null; + return token.text; } - /** - * Returns a sequence of remaining tokens in this, or nulls when none remain. - * This may fill the token buffer, but not otherwise modify it. - */ - public Supplier lookahead() { - Iterator iterator = tokens.iterator(); - if (iterator.hasNext()) iterator.next(); - return () -> iterator.hasNext() ? iterator.next() : null; + public int remaining() { + return tokens.size() - position; } private void add(JsonToken token, String name, String text) { - tokens.add(new Token(token, name, text)); + tokens.add(tokens.size(), new Token(token, name, text)); } - public void bufferObject(JsonParser parser) { - bufferJsonStruct(parser, JsonToken.START_OBJECT); + public void bufferObject(JsonToken first, JsonParser tokens) { + bufferJsonStruct(first, tokens, JsonToken.START_OBJECT); } - private void bufferJsonStruct(JsonParser parser, JsonToken firstToken) { - JsonToken token = parser.currentToken(); - Preconditions.checkArgument(token == firstToken, - "Expected %s, got %s.", firstToken.name(), token); - updateNesting(token); + private void bufferJsonStruct(JsonToken first, JsonParser tokens, JsonToken firstToken) { + int localNesting = 0; + JsonToken t = first; - try { - for (int nesting = addFromParser(parser); nesting > 0; nesting += addFromParser(parser)) - parser.nextValue(); + Preconditions.checkArgument(first == firstToken, + "Expected %s, got %s.", firstToken.name(), t); + if (remaining() == 0) { + updateNesting(t); } - catch (IOException e) { - throw new IllegalArgumentException(e); + localNesting = storeAndPeekNesting(t, localNesting, tokens); + while (localNesting > 0) { + t = nextValue(tokens); + localNesting = storeAndPeekNesting(t, localNesting, tokens); } } - int nestingOffset(JsonToken token) { + private int storeAndPeekNesting(JsonToken t, int nesting, JsonParser tokens) { + addFromParser(t, tokens); + return nesting + nestingOffset(t); + } + + private int nestingOffset(JsonToken token) { if (token == null) return 0; if (token.isStructStart()) { return 1; @@ -98,23 +122,43 @@ public class TokenBuffer { } } - int addFromParser(JsonParser tokens) throws IOException { - add(tokens.currentToken(), tokens.getCurrentName(), tokens.getText()); - return nestingOffset(tokens.currentToken()); + private void addFromParser(JsonToken t, JsonParser tokens) { + try { + add(t, tokens.getCurrentName(), tokens.getText()); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + private JsonToken nextValue(JsonParser tokens) { + try { + return tokens.nextValue(); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } } - void updateNesting(JsonToken token) { + private void updateNesting(JsonToken token) { nesting += nestingOffset(token); } + private void updateNestingGoingBackwards(JsonToken token) { + nesting -= nestingOffset(token); + } + public int nesting() { return nesting; } public void skipToRelativeNesting(int relativeNesting) { int initialNesting = nesting(); - do next(); - while (nesting() > initialNesting + relativeNesting); + do { + next(); + } while ( nesting() > initialNesting + relativeNesting); + } + + public List rest() { + return tokens.subList(position, tokens.size()); } public static final class Token { diff --git a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java index aef7e1cffe2..74656762fe1 100644 --- a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java +++ b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java @@ -86,6 +86,16 @@ public class DocumentParser { private void handleIdentLevelOne(DocumentParseInfo documentParseInfo, boolean docIdAndOperationIsSetExternally) throws IOException { JsonToken currentToken = parser.getCurrentToken(); + if (currentToken == JsonToken.VALUE_TRUE || currentToken == JsonToken.VALUE_FALSE) { + try { + if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { + documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue()); + return; + } + } catch (IOException e) { + throw new RuntimeException("Got IO exception while parsing document", e); + } + } if ((currentToken == JsonToken.VALUE_TRUE || currentToken == JsonToken.VALUE_FALSE) && CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { documentParseInfo.create = Optional.of(currentToken == JsonToken.VALUE_TRUE); @@ -101,11 +111,12 @@ public class DocumentParser { } } - private void handleIdentLevelTwo(DocumentParseInfo documentParseInfo) { + private void handleIdentLevelTwo(DocumentParseInfo documentParseInfo) { try { + JsonToken currentToken = parser.getCurrentToken(); // "fields" opens a dictionary and is therefore on level two which might be surprising. - if (parser.currentToken() == JsonToken.START_OBJECT && FIELDS.equals(parser.getCurrentName())) { - documentParseInfo.fieldsBuffer.bufferObject(parser); + if (currentToken == JsonToken.START_OBJECT && FIELDS.equals(parser.getCurrentName())) { + documentParseInfo.fieldsBuffer.bufferObject(currentToken, parser); processIndent(); } } catch (IOException e) { diff --git a/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java index e859306f04d..2dce07cdbe6 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java +++ b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java @@ -8,7 +8,6 @@ import com.yahoo.document.json.TokenBuffer; import java.util.Optional; public class DocumentParseInfo { - public DocumentParseInfo() { } public DocumentId documentId; public Optional create = Optional.empty(); public Optional condition = Optional.empty(); diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java index 1fd4029b1a5..0b7b1ae9996 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java @@ -4,15 +4,13 @@ package com.yahoo.document.json.readers; import com.fasterxml.jackson.core.JsonToken; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.json.TokenBuffer; -import com.yahoo.document.json.TokenBuffer.Token; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Type; import com.yahoo.tensor.IndexedTensor; import com.yahoo.tensor.MixedTensor; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; -import com.yahoo.tensor.TensorType.Dimension; - -import java.util.function.Supplier; import static com.yahoo.document.json.readers.JsonParserHelpers.*; import static com.yahoo.tensor.serialization.JsonFormat.decodeHexString; @@ -39,43 +37,36 @@ public class TensorReader { Tensor.Builder builder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType()); expectOneOf(buffer.current(), JsonToken.START_OBJECT, JsonToken.START_ARRAY); int initNesting = buffer.nesting(); - while (true) { - Supplier lookahead = buffer.lookahead(); - Token next = lookahead.get(); - if (TENSOR_CELLS.equals(next.name) && ! primitiveContent(next.token, lookahead.get().token)) { - buffer.next(); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { + if (TENSOR_CELLS.equals(buffer.currentName()) && ! primitiveContent(buffer)) { readTensorCells(buffer, builder); } - else if (TENSOR_VALUES.equals(next.name) && builder.type().dimensions().stream().allMatch(Dimension::isIndexed)) { - buffer.next(); + else if (TENSOR_VALUES.equals(buffer.currentName()) && builder.type().dimensions().stream().allMatch(d -> d.isIndexed())) { readTensorValues(buffer, builder); } - else if (TENSOR_BLOCKS.equals(next.name)) { - buffer.next(); + else if (TENSOR_BLOCKS.equals(buffer.currentName())) { readTensorBlocks(buffer, builder); } - else if (TENSOR_TYPE.equals(next.name) && next.token == JsonToken.VALUE_STRING) { - buffer.next(); + else if (TENSOR_TYPE.equals(buffer.currentName()) && buffer.current() == JsonToken.VALUE_STRING) { // Ignore input tensor type } - else if (buffer.nesting() == initNesting && JsonToken.END_OBJECT == next.token) { - buffer.next(); - break; - } else { + buffer.previous(); // Back up to the start of the enclosing block readDirectTensorValue(buffer, builder); - break; + buffer.previous(); // ... and back up to the end of the enclosing block } } expectOneOf(buffer.current(), JsonToken.END_OBJECT, JsonToken.END_ARRAY); tensorFieldValue.assign(builder.build()); } - static boolean primitiveContent(JsonToken current, JsonToken next) { - if (current.isScalarValue()) return true; - if (current == JsonToken.START_ARRAY) { - if (next == JsonToken.END_ARRAY) return false; - if (next.isScalarValue()) return true; + static boolean primitiveContent(TokenBuffer buffer) { + JsonToken cellsValue = buffer.current(); + if (cellsValue.isScalarValue()) return true; + if (cellsValue == JsonToken.START_ARRAY) { + JsonToken firstArrayValue = buffer.peek(1); + if (firstArrayValue == JsonToken.END_ARRAY) return false; + if (firstArrayValue.isScalarValue()) return true; } return false; } @@ -195,7 +186,7 @@ public class TensorReader { boolean hasIndexed = builder.type().dimensions().stream().anyMatch(TensorType.Dimension::isIndexed); boolean hasMapped = builder.type().dimensions().stream().anyMatch(TensorType.Dimension::isMapped); - if (isArrayOfObjects(buffer)) + if (isArrayOfObjects(buffer, 0)) readTensorCells(buffer, builder); else if ( ! hasMapped) readTensorValues(buffer, builder); @@ -205,12 +196,10 @@ public class TensorReader { readTensorCells(buffer, builder); } - private static boolean isArrayOfObjects(TokenBuffer buffer) { - if (buffer.current() != JsonToken.START_ARRAY) return false; - Supplier lookahead = buffer.lookahead(); - Token next; - while ((next = lookahead.get()).token == JsonToken.START_ARRAY) { } - return next.token == JsonToken.START_OBJECT; + private static boolean isArrayOfObjects(TokenBuffer buffer, int ahead) { + if (buffer.peek(ahead++) != JsonToken.START_ARRAY) return false; + if (buffer.peek(ahead) == JsonToken.START_ARRAY) return isArrayOfObjects(buffer, ahead); // nested array + return buffer.peek(ahead) == JsonToken.START_OBJECT; } private static TensorAddress readAddress(TokenBuffer buffer, TensorType type) { diff --git a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java index c7303d31ea2..113b8732b23 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java @@ -238,7 +238,7 @@ public class VespaJsonDocumentReader { "Expected end of JSON struct (%s), got %s", expectedFinalToken, buffer.current()); Preconditions.checkState(buffer.nesting() == 0, "Nesting not zero at end of operation"); Preconditions.checkState(buffer.next() == null, "Dangling data at end of operation"); - Preconditions.checkState(buffer.isEmpty(), "Dangling data at end of operation"); + Preconditions.checkState(buffer.remaining() == 0, "Dangling data at end of operation"); } } diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 080528fea77..5a9f02c790d 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -2120,93 +2120,69 @@ public class JsonReaderTestCase { @Test public void tensor_modify_update_with_replace_operation() { assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.REPLACE, "sparse_tensor", - """ - { - "operation": "replace", - "cells": [ - { "address": { "x": "a", "y": "b" }, "value": 2.0 } - ] - }"""); + inputJson("{", + " 'operation': 'replace',", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}")); } @Test public void tensor_modify_update_with_add_operation() { assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.ADD, "sparse_tensor", - """ - { - "operation": "add", - "cells": [ - { "address": { "x": "a", "y": "b" }, "value": 2.0 } - ] - }"""); + inputJson("{", + " 'operation': 'add',", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}")); } @Test public void tensor_modify_update_with_multiply_operation() { assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.MULTIPLY, "sparse_tensor", - """ - { - "operation": "multiply", - "cells": [ - { "address": { "x": "a", "y": "b" }, "value": 2.0 } - ] - }"""); + inputJson("{", + " 'operation': 'multiply',", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}")); } @Test public void tensor_modify_update_with_create_non_existing_cells_true() { assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.ADD, true, "sparse_tensor", - """ - { - "operation": "add", - "create": true, - "cells": [ - { "address": { "x": "a", "y": "b" }, "value": 2.0 } - ] - }"""); + inputJson("{", + " 'operation': 'add',", + " 'create': true,", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}")); } @Test public void tensor_modify_update_with_create_non_existing_cells_false() { assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.ADD, false, "sparse_tensor", - """ - { - "operation": "add", - "create": false, - "cells": [ - { "address": { "x": "a", "y": "b" }, "value": 2.0 } - ] - }"""); + inputJson("{", + " 'operation': 'add',", + " 'create': false,", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}")); } @Test public void tensor_modify_update_treats_the_input_tensor_as_sparse() { // Note that the type of the tensor in the modify update is sparse (it only has mapped dimensions). assertTensorModifyUpdate("tensor(x{},y{}):{{x:0,y:0}:2.0, {x:1,y:2}:3.0}", - TensorModifyUpdate.Operation.REPLACE, "dense_tensor", - """ - { - "operation": "replace", - "cells": [ - { "address": { "x": "0", "y": "0" }, "value": 2.0 }, - { "address": { "x": "1", "y": "2" }, "value": 3.0 } - ] - }"""); + TensorModifyUpdate.Operation.REPLACE, "dense_tensor", + inputJson("{", + " 'operation': 'replace',", + " 'cells': [", + " { 'address': { 'x': '0', 'y': '0' }, 'value': 2.0 },", + " { 'address': { 'x': '1', 'y': '2' }, 'value': 3.0 } ]}")); } @Test public void tensor_modify_update_on_non_tensor_field_throws() { try { - JsonReader reader = createReader(""" - { - "update": "id:unittest:smoke::doc1", - "fields": { - "something": { - "modify": {} - } - } - } - """); + JsonReader reader = createReader(inputJson("{ 'update': 'id:unittest:smoke::doc1',", + " 'fields': {", + " 'something': {", + " 'modify': {} }}}")); reader.readSingleDocument(DocumentOperationType.UPDATE, "id:unittest:smoke::doc1"); fail("Expected exception"); } @@ -2220,125 +2196,95 @@ public class JsonReaderTestCase { public void tensor_modify_update_on_dense_unbound_tensor_throws() { illegalTensorModifyUpdate("Error in 'dense_unbound_tensor': A modify update cannot be applied to tensor types with indexed unbound dimensions. Field 'dense_unbound_tensor' has unsupported tensor type 'tensor(x[],y[])'", "dense_unbound_tensor", - """ - { - "operation": "replace", - "cells": [ - { "address": { "x": "0", "y": "0" }, "value": 2.0 } - ] - }"""); + "{", + " 'operation': 'replace',", + " 'cells': [", + " { 'address': { 'x': '0', 'y': '0' }, 'value': 2.0 } ]}"); } @Test public void tensor_modify_update_on_sparse_tensor_with_single_dimension_short_form() { - assertTensorModifyUpdate("{{x:a}:2.0, {x:c}: 3.0}", TensorModifyUpdate.Operation.REPLACE, "sparse_single_dimension_tensor", - """ - { - "operation": "replace", - "cells": { - "a": 2.0, - "c": 3.0 - } - }"""); + assertTensorModifyUpdate("{{x:a}:2.0, {x:c}: 3.0}", TensorModifyUpdate.Operation.REPLACE, "sparse_single_dimension_tensor", + inputJson("{", + " 'operation': 'replace',", + " 'cells': {", + " 'a': 2.0,", + " 'c': 3.0 }}")); } @Test public void tensor_modify_update_with_replace_operation_mixed() { assertTensorModifyUpdate("{{x:a,y:0}:2.0}", TensorModifyUpdate.Operation.REPLACE, "mixed_tensor", - """ - { - "operation": "replace", - "cells": [ - { "address": { "x": "a", "y": "0" }, "value": 2.0 } - ] - }"""); + inputJson("{", + " 'operation': 'replace',", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': '0' }, 'value': 2.0 } ]}")); } @Test public void tensor_modify_update_with_replace_operation_mixed_block_short_form_array() { assertTensorModifyUpdate("{{x:a,y:0}:1,{x:a,y:1}:2,{x:a,y:2}:3}", TensorModifyUpdate.Operation.REPLACE, "mixed_tensor", - """ - { - "operation": "replace", - "blocks": [ - { "address": { "x": "a" }, "values": [1,2,3] } - ] - }"""); + inputJson("{", + " 'operation': 'replace',", + " 'blocks': [", + " { 'address': { 'x': 'a' }, 'values': [1,2,3] } ]}")); } @Test public void tensor_modify_update_with_replace_operation_mixed_block_short_form_must_specify_full_subspace() { illegalTensorModifyUpdate("Error in 'mixed_tensor': At {x:a}: Expected 3 values, but got 2", - "mixed_tensor", - """ - { - "operation": "replace", - "blocks": { - "a": [2,3] - } - }"""); + "mixed_tensor", + inputJson("{", + " 'operation': 'replace',", + " 'blocks': {", + " 'a': [2,3] } }")); } @Test public void tensor_modify_update_with_replace_operation_mixed_block_short_form_map() { assertTensorModifyUpdate("{{x:a,y:0}:1,{x:a,y:1}:2,{x:a,y:2}:3}", TensorModifyUpdate.Operation.REPLACE, "mixed_tensor", - """ - { - "operation": "replace", - "blocks": { - "a": [1,2,3] - } - }"""); + inputJson("{", + " 'operation': 'replace',", + " 'blocks': {", + " 'a': [1,2,3] } }")); } @Test public void tensor_modify_update_with_add_operation_mixed() { assertTensorModifyUpdate("{{x:a,y:0}:2.0}", TensorModifyUpdate.Operation.ADD, "mixed_tensor", - """ - { - "operation": "add", - "cells": [ - { "address": { "x": "a", "y": "0" }, "value": 2.0 } - ] - }"""); + inputJson("{", + " 'operation': 'add',", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': '0' }, 'value': 2.0 } ]}")); } @Test public void tensor_modify_update_with_multiply_operation_mixed() { assertTensorModifyUpdate("{{x:a,y:0}:2.0}", TensorModifyUpdate.Operation.MULTIPLY, "mixed_tensor", - """ - { - "operation": "multiply", - "cells": [ - { "address": { "x": "a", "y": "0" }, "value": 2.0 } - ] - }"""); + inputJson("{", + " 'operation': 'multiply',", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': '0' }, 'value': 2.0 } ]}")); } @Test public void tensor_modify_update_with_out_of_bound_cells_throws() { illegalTensorModifyUpdate("Error in 'dense_tensor': Dimension 'y' has label '3' but type is tensor(x[2],y[3])", "dense_tensor", - """ - { - "operation": "replace", - "cells": [ - { "address": { "x": "0", "y": "3" }, "value": 2.0 } - ] - }"""); + "{", + " 'operation': 'replace',", + " 'cells': [", + " { 'address': { 'x': '0', 'y': '3' }, 'value': 2.0 } ]}"); } @Test public void tensor_modify_update_with_out_of_bound_cells_throws_mixed() { illegalTensorModifyUpdate("Error in 'mixed_tensor': Dimension 'y' has label '3' but type is tensor(x{},y[3])", "mixed_tensor", - """ - { - "operation": "replace", - "cells": [ - { "address": { "x": "0", "y": "3" }, "value": 2.0 } - ] - }"""); + "{", + " 'operation': 'replace',", + " 'cells': [", + " { 'address': { 'x': '0', 'y': '3' }, 'value': 2.0 } ]}"); } @@ -2346,113 +2292,87 @@ public class JsonReaderTestCase { public void tensor_modify_update_with_unknown_operation_throws() { illegalTensorModifyUpdate("Error in 'sparse_tensor': Unknown operation 'unknown' in modify update for field 'sparse_tensor'", "sparse_tensor", - """ - { - "operation": "unknown", - "cells": [ - { "address": { "x": "a", "y": "b" }, "value": 2.0 } - ] - }"""); + "{", + " 'operation': 'unknown',", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}"); } @Test public void tensor_modify_update_without_operation_throws() { illegalTensorModifyUpdate("Error in 'sparse_tensor': Modify update for field 'sparse_tensor' does not contain an operation", "sparse_tensor", - """ - { - "cells": [] - }"""); + "{", + " 'cells': [] }"); } @Test public void tensor_modify_update_without_cells_throws() { illegalTensorModifyUpdate("Error in 'sparse_tensor': Modify update for field 'sparse_tensor' does not contain tensor cells", "sparse_tensor", - """ - { - "operation": "replace" - }"""); + "{", + " 'operation': 'replace' }"); } @Test public void tensor_modify_update_with_unknown_content_throws() { illegalTensorModifyUpdate("Error in 'sparse_tensor': Unknown JSON string 'unknown' in modify update for field 'sparse_tensor'", "sparse_tensor", - """ - { - "unknown": "here" - }"""); + "{", + " 'unknown': 'here' }"); } @Test public void tensor_add_update_on_sparse_tensor() { assertTensorAddUpdate("{{x:a,y:b}:2.0, {x:c,y:d}: 3.0}", "sparse_tensor", - """ - { - "cells": [ - { "address": { "x": "a", "y": "b" }, "value": 2.0 }, - { "address": { "x": "c", "y": "d" }, "value": 3.0 } - ] - }"""); + inputJson("{", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 },", + " { 'address': { 'x': 'c', 'y': 'd' }, 'value': 3.0 } ]}")); } @Test public void tensor_add_update_on_sparse_tensor_with_single_dimension_short_form() { assertTensorAddUpdate("{{x:a}:2.0, {x:c}: 3.0}", "sparse_single_dimension_tensor", - """ - { - "cells": { - "a": 2.0, - "c": 3.0 - } - }"""); + inputJson("{", + " 'cells': {", + " 'a': 2.0,", + " 'c': 3.0 }}")); } @Test public void tensor_add_update_on_mixed_tensor() { assertTensorAddUpdate("{{x:a,y:0}:2.0, {x:a,y:1}:3.0, {x:a,y:2}:0.0}", "mixed_tensor", - """ - { - "cells": [ - { "address": { "x": "a", "y": "0" }, "value": 2.0 }, - { "address": { "x": "a", "y": "1" }, "value": 3.0 } - ] - }"""); + inputJson("{", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': '0' }, 'value': 2.0 },", + " { 'address': { 'x': 'a', 'y': '1' }, 'value': 3.0 } ]}")); } @Test public void tensor_add_update_on_mixed_with_out_of_bound_dense_cells_throws() { illegalTensorAddUpdate("Error in 'mixed_tensor': Index 3 out of bounds for length 3", "mixed_tensor", - """ - { - "cells": [ - { "address": { "x": "0", "y": "3" }, "value": 2.0 } - ] - }"""); + "{", + " 'cells': [", + " { 'address': { 'x': '0', 'y': '3' }, 'value': 2.0 } ]}"); } @Test public void tensor_add_update_on_dense_tensor_throws() { illegalTensorAddUpdate("Error in 'dense_tensor': An add update can only be applied to tensors with at least one sparse dimension. Field 'dense_tensor' has unsupported tensor type 'tensor(x[2],y[3])'", "dense_tensor", - """ - { - "cells": [ ] - }"""); + "{", + " 'cells': [] }"); } @Test public void tensor_add_update_on_not_fully_specified_cell_throws() { illegalTensorAddUpdate("Error in 'sparse_tensor': Missing a label for dimension 'y' for tensor(x{},y{})", "sparse_tensor", - """ - { - "cells": [ - { "address": { "x": "a" }, "value": 2.0 } - ] - }"""); + "{", + " 'cells': [", + " { 'address': { 'x': 'a' }, 'value': 2.0 } ]}"); } @Test @@ -2468,176 +2388,146 @@ public class JsonReaderTestCase { @Test public void tensor_remove_update_on_sparse_tensor() { assertTensorRemoveUpdate("{{x:a,y:b}:1.0,{x:c,y:d}:1.0}", "sparse_tensor", - """ - { - "addresses": [ - { "x": "a", "y": "b" }, - { "x": "c", "y": "d" } - ] - }"""); + inputJson("{", + " 'addresses': [", + " { 'x': 'a', 'y': 'b' },", + " { 'x': 'c', 'y': 'd' } ]}")); } @Test public void tensor_remove_update_on_mixed_tensor() { assertTensorRemoveUpdate("{{x:1}:1.0,{x:2}:1.0}", "mixed_tensor", - """ - { - "addresses": [ - { "x": "1" }, - { "x": "2" } - ] - }"""); + inputJson("{", + " 'addresses': [", + " { 'x': '1' },", + " { 'x': '2' } ]}")); } @Test public void tensor_remove_update_on_sparse_tensor_with_not_fully_specified_address() { assertTensorRemoveUpdate("{{y:b}:1.0,{y:d}:1.0}", "sparse_tensor", - """ - { - "addresses": [ - { "y": "b" }, - { "y": "d" } - ] - }"""); + inputJson("{", + " 'addresses': [", + " { 'y': 'b' },", + " { 'y': 'd' } ]}")); } @Test public void tensor_remove_update_on_mixed_tensor_with_not_fully_specified_address() { assertTensorRemoveUpdate("{{x:1,z:a}:1.0,{x:2,z:b}:1.0}", "mixed_tensor_adv", - """ - { - "addresses": [ - { "x": "1", "z": "a" }, - { "x": "2", "z": "b" } - ] - }"""); + inputJson("{", + " 'addresses': [", + " { 'x': '1', 'z': 'a' },", + " { 'x': '2', 'z': 'b' } ]}")); } @Test public void tensor_remove_update_on_mixed_tensor_with_dense_addresses_throws() { illegalTensorRemoveUpdate("Error in 'mixed_tensor': Indexed dimension address 'y' should not be specified in remove update", "mixed_tensor", - """ - { - "addresses": [ - { "x": "1", "y": "0" }, - { "x": "2", "y": "0" } - ] - }"""); + "{", + " 'addresses': [", + " { 'x': '1', 'y': '0' },", + " { 'x': '2', 'y': '0' } ]}"); } @Test public void tensor_remove_update_on_dense_tensor_throws() { illegalTensorRemoveUpdate("Error in 'dense_tensor': A remove update can only be applied to tensors with at least one sparse dimension. Field 'dense_tensor' has unsupported tensor type 'tensor(x[2],y[3])'", "dense_tensor", - """ - { - "addresses": [] - }"""); + "{", + " 'addresses': [] }"); } @Test public void tensor_remove_update_with_stray_dimension_throws() { illegalTensorRemoveUpdate("Error in 'sparse_tensor': tensor(x{},y{}) does not contain dimension 'foo'", - "sparse_tensor", - """ - { - "addresses": [ - { "x": "a", "foo": "b" } - ] - }"""); + "sparse_tensor", + "{", + " 'addresses': [", + " { 'x': 'a', 'foo': 'b' } ]}"); illegalTensorRemoveUpdate("Error in 'sparse_tensor': tensor(x{}) does not contain dimension 'foo'", - "sparse_tensor", - """ - { - "addresses": [ - { "x": "c" }, - { "x": "a", "foo": "b" } - ] - }"""); + "sparse_tensor", + "{", + " 'addresses': [", + " { 'x': 'c' },", + " { 'x': 'a', 'foo': 'b' } ]}"); } @Test public void tensor_remove_update_without_cells_throws() { illegalTensorRemoveUpdate("Error in 'sparse_tensor': Remove update for field 'sparse_tensor' does not contain tensor addresses", "sparse_tensor", - """ - { - "addresses": [] - }"""); + "{'addresses': [] }"); illegalTensorRemoveUpdate("Error in 'mixed_tensor': Remove update for field 'mixed_tensor' does not contain tensor addresses", "mixed_tensor", - """ - { - "addresses": [] - }"""); + "{'addresses': [] }"); } @Test public void require_that_parser_propagates_datatype_parser_errors_predicate() { assertParserErrorMatches( "Error in document 'id:unittest:testpredicate::0' - could not parse field 'boolean' of type 'predicate': " + - "line 1:10 no viable alternative at character '>'", - """ - [ - { - "fields": { - "boolean": "timestamp > 9000" - }, - "put": "id:unittest:testpredicate::0" - } - ] - """); + "line 1:10 no viable alternative at character '>'", + + "[", + " {", + " 'fields': {", + " 'boolean': 'timestamp > 9000'", + " },", + " 'put': 'id:unittest:testpredicate::0'", + " }", + "]" + ); } @Test public void require_that_parser_propagates_datatype_parser_errors_string_as_int() { assertParserErrorMatches( "Error in document 'id:unittest:testint::0' - could not parse field 'integerfield' of type 'int': " + - "For input string: \" 1\"", - """ - [ - { - "fields": { - "integerfield": " 1" - }, - "put": "id:unittest:testint::0" - } - ] - """); + "For input string: \" 1\"", + + "[", + " {", + " 'fields': {", + " 'integerfield': ' 1'", + " },", + " 'put': 'id:unittest:testint::0'", + " }", + "]" + ); } @Test public void require_that_parser_propagates_datatype_parser_errors_overflowing_int() { assertParserErrorMatches( "Error in document 'id:unittest:testint::0' - could not parse field 'integerfield' of type 'int': " + - "For input string: \"281474976710656\"", - """ - [ - { - "fields": { - "integerfield": 281474976710656 - }, - "put": "id:unittest:testint::0" - } - ] - """); + "For input string: \"281474976710656\"", + + "[", + " {", + " 'fields': {", + " 'integerfield': 281474976710656", + " },", + " 'put': 'id:unittest:testint::0'", + " }", + "]" + ); } @Test public void requireThatUnknownDocTypeThrowsIllegalArgumentException() { - String jsonData = """ - [ - { - "put": "id:ns:walrus::walrus1", - "fields": { - "aField": 42 - } - } - ] - """; + final String jsonData = inputJson( + "[", + " {", + " 'put': 'id:ns:walrus::walrus1',", + " 'fields': {", + " 'aField': 42", + " }", + " }", + "]"); try { new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next(); fail(); @@ -2687,40 +2577,30 @@ public class JsonReaderTestCase { return createPutWithTensor(inputTensor, "sparse_tensor"); } private DocumentPut createPutWithTensor(String inputTensor, String tensorFieldName) { - JsonReader streaming = createReader(""" - { - "fields": { - "%s": %s - } - } - """.formatted(tensorFieldName, inputTensor)); - DocumentPut lazyParsed = (DocumentPut) streaming.readSingleDocumentStreaming(DocumentOperationType.PUT, TENSOR_DOC_ID).operation(); - JsonReader reader = createReader(""" - [ - { - "put": "%s", - "fields": { - "%s": %s - } - } - ]""".formatted(TENSOR_DOC_ID, tensorFieldName, inputTensor)); - DocumentPut bufferParsed = (DocumentPut) reader.next(); - assertEquals(lazyParsed, bufferParsed); - return bufferParsed; + JsonReader reader = createReader(inputJson("[", + "{ 'put': '" + TENSOR_DOC_ID + "',", + " 'fields': {", + " '" + tensorFieldName + "': " + inputTensor + " }}]")); + return (DocumentPut) reader.next(); } private DocumentUpdate createAssignUpdateWithSparseTensor(String inputTensor) { return createAssignUpdateWithTensor(inputTensor, "sparse_tensor"); } private DocumentUpdate createAssignUpdateWithTensor(String inputTensor, String tensorFieldName) { - return createTensorUpdate("assign", inputTensor, tensorFieldName); + JsonReader reader = createReader(inputJson("[", + "{ 'update': '" + TENSOR_DOC_ID + "',", + " 'fields': {", + " '" + tensorFieldName + "': {", + " 'assign': " + (inputTensor != null ? inputTensor : "null") + " } } } ]")); + return (DocumentUpdate) reader.next(); } private static Tensor assertSparseTensorField(String expectedTensor, DocumentPut put) { return assertTensorField(expectedTensor, put, "sparse_tensor"); } private Tensor assertTensorField(String expectedTensor, String fieldName, String inputJson) { - return assertTensorField(expectedTensor, createPutWithTensor(inputJson(inputJson), fieldName), fieldName); + return assertTensorField(expectedTensor, createPutWithTensor(inputJson, fieldName), fieldName); } private static Tensor assertTensorField(String expectedTensor, DocumentPut put, String tensorFieldName) { return assertTensorField(Tensor.from(expectedTensor), put, tensorFieldName); @@ -2793,29 +2673,12 @@ public class JsonReaderTestCase { } private DocumentUpdate createTensorUpdate(String operation, String tensorJson, String tensorFieldName) { - JsonReader streaming = createReader(""" - { - "fields": { - "%s": { - "%s": %s - } - } - }""".formatted(tensorFieldName, operation, tensorJson)); - DocumentUpdate lazyParsed = (DocumentUpdate) streaming.readSingleDocumentStreaming(DocumentOperationType.UPDATE, TENSOR_DOC_ID).operation(); - JsonReader reader = createReader(""" - [ - { - "update": "%s", - "fields": { - "%s": { - "%s": %s - } - } - } - ]""".formatted(TENSOR_DOC_ID, tensorFieldName, operation, tensorJson)); - DocumentUpdate bufferParsed = (DocumentUpdate) reader.next(); - assertEquals(lazyParsed, bufferParsed); - return bufferParsed; + JsonReader reader = createReader(inputJson("[", + "{ 'update': '" + TENSOR_DOC_ID + "',", + " 'fields': {", + " '" + tensorFieldName + "': {", + " '" + operation + "': " + tensorJson + " }}}]")); + return (DocumentUpdate) reader.next(); } private void assertTensorAddUpdate(String expectedTensor, String tensorFieldName, String tensorJson) { diff --git a/document/src/test/java/com/yahoo/document/json/LazyTokenBufferTest.java b/document/src/test/java/com/yahoo/document/json/LazyTokenBufferTest.java deleted file mode 100644 index 3ed2ed531c3..00000000000 --- a/document/src/test/java/com/yahoo/document/json/LazyTokenBufferTest.java +++ /dev/null @@ -1,132 +0,0 @@ -package com.yahoo.document.json; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import com.yahoo.document.json.TokenBuffer.Token; -import org.junit.Test; - -import java.io.IOException; -import java.util.function.Supplier; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -/** - * @author jonmv - */ -public class LazyTokenBufferTest { - - @Test - public void testBuffer() throws IOException { - String json = """ - { - "fields": { - "foo": "bar", - "baz": [1, 2, 3], - "quu": { "qux": null } - } - }"""; - JsonParser parser = new JsonFactory().createParser(json); - parser.nextValue(); - parser.nextValue(); - assertEquals(JsonToken.START_OBJECT, parser.currentToken()); - assertEquals("fields", parser.currentName()); - - // Peeking through the buffer doesn't change nesting. - LazyTokenBuffer buffer = new LazyTokenBuffer(parser); - assertEquals(JsonToken.START_OBJECT, buffer.current()); - assertEquals("fields", buffer.currentName()); - assertEquals(1, buffer.nesting()); - - Supplier lookahead = buffer.lookahead(); - Token peek = lookahead.get(); - assertEquals(JsonToken.VALUE_STRING, peek.token); - assertEquals("foo", peek.name); - assertEquals("bar", peek.text); - assertEquals(1, buffer.nesting()); - - peek = lookahead.get(); - assertEquals(JsonToken.START_ARRAY, peek.token); - assertEquals("baz", peek.name); - assertEquals(1, buffer.nesting()); - - peek = lookahead.get(); - assertEquals(JsonToken.VALUE_NUMBER_INT, peek.token); - assertEquals("1", peek.text); - - peek = lookahead.get(); - assertEquals(JsonToken.VALUE_NUMBER_INT, peek.token); - assertEquals("2", peek.text); - - peek = lookahead.get(); - assertEquals(JsonToken.VALUE_NUMBER_INT, peek.token); - assertEquals("3", peek.text); - - peek = lookahead.get(); - assertEquals(JsonToken.END_ARRAY, peek.token); - assertEquals(1, buffer.nesting()); - - peek = lookahead.get(); - assertEquals(JsonToken.START_OBJECT, peek.token); - assertEquals("quu", peek.name); - assertEquals(1, buffer.nesting()); - - peek = lookahead.get(); - assertEquals(JsonToken.VALUE_NULL, peek.token); - assertEquals("qux", peek.name); - - peek = lookahead.get(); - assertEquals(JsonToken.END_OBJECT, peek.token); - assertEquals(1, buffer.nesting()); - - peek = lookahead.get(); - assertEquals(JsonToken.END_OBJECT, peek.token); - assertEquals(1, buffer.nesting()); - - peek = lookahead.get(); - assertNull(peek); - - // Parser is now at the end. - assertEquals(JsonToken.END_OBJECT, parser.nextToken()); - assertNull(parser.nextToken()); - - // Repeat iterating through the buffer, this time advancing it, and see that nesting changes. - assertEquals(JsonToken.VALUE_STRING, buffer.next()); - assertEquals("foo", buffer.currentName()); - assertEquals("bar", buffer.currentText()); - assertEquals(1, buffer.nesting()); - - assertEquals(JsonToken.START_ARRAY, buffer.next()); - assertEquals("baz", buffer.currentName()); - assertEquals(2, buffer.nesting()); - - assertEquals(JsonToken.VALUE_NUMBER_INT, buffer.next()); - assertEquals("1", buffer.currentText()); - - assertEquals(JsonToken.VALUE_NUMBER_INT, buffer.next()); - assertEquals("2", buffer.currentText()); - - assertEquals(JsonToken.VALUE_NUMBER_INT, buffer.next()); - assertEquals("3", buffer.currentText()); - - assertEquals(JsonToken.END_ARRAY, buffer.next()); - assertEquals(1, buffer.nesting()); - - assertEquals(JsonToken.START_OBJECT, buffer.next()); - assertEquals("quu", buffer.currentName()); - assertEquals(2, buffer.nesting()); - - assertEquals(JsonToken.VALUE_NULL, buffer.next()); - assertEquals("qux", buffer.currentName()); - - assertEquals(JsonToken.END_OBJECT, buffer.next()); - assertEquals(1, buffer.nesting()); - - assertEquals(JsonToken.END_OBJECT, buffer.next()); - assertEquals(0, buffer.nesting()); - - assertNull(buffer.next()); - } - -} -- cgit v1.2.3