diff options
author | jonmv <venstad@gmail.com> | 2024-01-23 16:03:58 +0100 |
---|---|---|
committer | jonmv <venstad@gmail.com> | 2024-01-23 16:49:25 +0100 |
commit | 0116af34b503a8b7f7a4d8e5585238883a746c32 (patch) | |
tree | e44f1441030e52ac14ac3bac9cbed1931cd82605 /document | |
parent | 914b47b6f43a86933b97077f995f8284cd1a8d86 (diff) |
Remove backwards token buffer APIs
Diffstat (limited to 'document')
4 files changed, 62 insertions, 79 deletions
diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java index dec84e46b77..5a8310e5d30 100644 --- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java +++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java @@ -1,15 +1,16 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.json; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; import com.google.common.base.Preconditions; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.function.Supplier; + /** * Helper class to enable lookahead in the token stream. * @@ -17,73 +18,52 @@ import com.google.common.base.Preconditions; */ public class TokenBuffer { - private final List<Token> tokens; + private final Deque<Token> tokens; - private int position = 0; private int nesting = 0; public TokenBuffer() { - this(new ArrayList<>()); - } - - public TokenBuffer(List<Token> tokens) { - this.tokens = tokens; - if (tokens.size() > 0) - updateNesting(tokens.get(position).token); + this.tokens = new ArrayDeque<>(); } /** Returns whether any tokens are available in this */ - public boolean isEmpty() { return remaining() == 0; } - - public JsonToken previous() { - updateNestingGoingBackwards(current()); - position--; - return current(); - } - - /** Returns the current token without changing position, or null if none */ - public JsonToken current() { - if (isEmpty()) return null; - Token token = tokens.get(position); - if (token == null) return null; - return token.token; - } + public boolean isEmpty() { return tokens.isEmpty(); } + /** Returns the next token, or null, and updates the nesting count of this. */ public JsonToken next() { - position++; + tokens.poll(); JsonToken token = current(); updateNesting(token); return token; } - /** Returns a given number of tokens ahead, or null if none */ - public JsonToken peek(int ahead) { - if (tokens.size() <= position + ahead) return null; - return tokens.get(position + ahead).token; + /** Returns the current token without changing position, or null if none */ + public JsonToken current() { + return isEmpty() ? null : tokens.peek().token; } /** Returns the current token name without changing position, or null if none */ public String currentName() { - if (isEmpty()) return null; - Token token = tokens.get(position); - if (token == null) return null; - return token.name; + return isEmpty() ? null : tokens.peek().name; } /** Returns the current token text without changing position, or null if none */ public String currentText() { - if (isEmpty()) return null; - Token token = tokens.get(position); - if (token == null) return null; - return token.text; + return isEmpty() ? null : tokens.peek().text; } - public int remaining() { - return tokens.size() - position; + /** + * Returns a sequence of remaining tokens in this, or nulls when none remain. + * This may fill the token buffer, but not otherwise modify it. + */ + public Supplier<Token> lookahead() { + Iterator<Token> iterator = tokens.iterator(); + if (iterator.hasNext()) iterator.next(); + return () -> iterator.hasNext() ? iterator.next() : null; } private void add(JsonToken token, String name, String text) { - tokens.add(tokens.size(), new Token(token, name, text)); + tokens.add(new Token(token, name, text)); } public void bufferObject(JsonToken first, JsonParser tokens) { @@ -96,7 +76,7 @@ public class TokenBuffer { Preconditions.checkArgument(first == firstToken, "Expected %s, got %s.", firstToken.name(), t); - if (remaining() == 0) { + if (isEmpty()) { updateNesting(t); } localNesting = storeAndPeekNesting(t, localNesting, tokens); @@ -142,23 +122,14 @@ public class TokenBuffer { nesting += nestingOffset(token); } - private void updateNestingGoingBackwards(JsonToken token) { - nesting -= nestingOffset(token); - } - public int nesting() { return nesting; } public void skipToRelativeNesting(int relativeNesting) { int initialNesting = nesting(); - do { - next(); - } while ( nesting() > initialNesting + relativeNesting); - } - - public List<Token> rest() { - return tokens.subList(position, tokens.size()); + do next(); + while (nesting() > initialNesting + relativeNesting); } public static final class Token { diff --git a/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java index 2dce07cdbe6..e859306f04d 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java +++ b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java @@ -8,6 +8,7 @@ import com.yahoo.document.json.TokenBuffer; import java.util.Optional; public class DocumentParseInfo { + public DocumentParseInfo() { } public DocumentId documentId; public Optional<Boolean> create = Optional.empty(); public Optional<String> condition = Optional.empty(); diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java index 0b7b1ae9996..27179ef32de 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java @@ -4,13 +4,15 @@ package com.yahoo.document.json.readers; import com.fasterxml.jackson.core.JsonToken; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.json.TokenBuffer; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Type; +import com.yahoo.document.json.TokenBuffer.Token; import com.yahoo.tensor.IndexedTensor; import com.yahoo.tensor.MixedTensor; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.TensorType.Dimension; + +import java.util.function.Supplier; import static com.yahoo.document.json.readers.JsonParserHelpers.*; import static com.yahoo.tensor.serialization.JsonFormat.decodeHexString; @@ -37,36 +39,43 @@ public class TensorReader { Tensor.Builder builder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType()); expectOneOf(buffer.current(), JsonToken.START_OBJECT, JsonToken.START_ARRAY); int initNesting = buffer.nesting(); - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { - if (TENSOR_CELLS.equals(buffer.currentName()) && ! primitiveContent(buffer)) { + while (true) { + Supplier<Token> lookahead = buffer.lookahead(); + Token next = lookahead.get(); + if (TENSOR_CELLS.equals(next.name) && ! primitiveContent(next.token, lookahead.get().token)) { + buffer.next(); readTensorCells(buffer, builder); } - else if (TENSOR_VALUES.equals(buffer.currentName()) && builder.type().dimensions().stream().allMatch(d -> d.isIndexed())) { + else if (TENSOR_VALUES.equals(next.name) && builder.type().dimensions().stream().allMatch(Dimension::isIndexed)) { + buffer.next(); readTensorValues(buffer, builder); } - else if (TENSOR_BLOCKS.equals(buffer.currentName())) { + else if (TENSOR_BLOCKS.equals(next.name)) { + buffer.next(); readTensorBlocks(buffer, builder); } - else if (TENSOR_TYPE.equals(buffer.currentName()) && buffer.current() == JsonToken.VALUE_STRING) { + else if (TENSOR_TYPE.equals(next.name) && next.token == JsonToken.VALUE_STRING) { + buffer.next(); // Ignore input tensor type } + else if (buffer.nesting() == initNesting && JsonToken.END_OBJECT == next.token) { + buffer.next(); + break; + } else { - buffer.previous(); // Back up to the start of the enclosing block readDirectTensorValue(buffer, builder); - buffer.previous(); // ... and back up to the end of the enclosing block + break; } } expectOneOf(buffer.current(), JsonToken.END_OBJECT, JsonToken.END_ARRAY); tensorFieldValue.assign(builder.build()); } - static boolean primitiveContent(TokenBuffer buffer) { - JsonToken cellsValue = buffer.current(); - if (cellsValue.isScalarValue()) return true; - if (cellsValue == JsonToken.START_ARRAY) { - JsonToken firstArrayValue = buffer.peek(1); - if (firstArrayValue == JsonToken.END_ARRAY) return false; - if (firstArrayValue.isScalarValue()) return true; + static boolean primitiveContent(JsonToken current, JsonToken next) { + if (current.isScalarValue()) return true; + if (current == JsonToken.START_ARRAY) { + if (next == JsonToken.END_ARRAY) return false; + if (next.isScalarValue()) return true; } return false; } @@ -186,7 +195,7 @@ public class TensorReader { boolean hasIndexed = builder.type().dimensions().stream().anyMatch(TensorType.Dimension::isIndexed); boolean hasMapped = builder.type().dimensions().stream().anyMatch(TensorType.Dimension::isMapped); - if (isArrayOfObjects(buffer, 0)) + if (isArrayOfObjects(buffer)) readTensorCells(buffer, builder); else if ( ! hasMapped) readTensorValues(buffer, builder); @@ -196,10 +205,12 @@ public class TensorReader { readTensorCells(buffer, builder); } - private static boolean isArrayOfObjects(TokenBuffer buffer, int ahead) { - if (buffer.peek(ahead++) != JsonToken.START_ARRAY) return false; - if (buffer.peek(ahead) == JsonToken.START_ARRAY) return isArrayOfObjects(buffer, ahead); // nested array - return buffer.peek(ahead) == JsonToken.START_OBJECT; + private static boolean isArrayOfObjects(TokenBuffer buffer) { + if (buffer.current() != JsonToken.START_ARRAY) return false; + Supplier<Token> lookahead = buffer.lookahead(); + Token next; + while ((next = lookahead.get()).token == JsonToken.START_ARRAY); + return next.token == JsonToken.START_OBJECT; } private static TensorAddress readAddress(TokenBuffer buffer, TensorType type) { diff --git a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java index 113b8732b23..c7303d31ea2 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java @@ -238,7 +238,7 @@ public class VespaJsonDocumentReader { "Expected end of JSON struct (%s), got %s", expectedFinalToken, buffer.current()); Preconditions.checkState(buffer.nesting() == 0, "Nesting not zero at end of operation"); Preconditions.checkState(buffer.next() == null, "Dangling data at end of operation"); - Preconditions.checkState(buffer.remaining() == 0, "Dangling data at end of operation"); + Preconditions.checkState(buffer.isEmpty(), "Dangling data at end of operation"); } } |