Remove backwards token buffer APIs

4 files changed, 62 insertions, 79 deletions

diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java
index dec84e46b77..5a8310e5d30 100644
--- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java
+++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java
@@ -1,15 +1,16 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.document.json;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonToken;
 import com.google.common.base.Preconditions;
 
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.function.Supplier;
+
 /**
  * Helper class to enable lookahead in the token stream.
  *
@@ -17,73 +18,52 @@ import com.google.common.base.Preconditions;
  */
 public class TokenBuffer {
 
-    private final List<Token> tokens;
+    private final Deque<Token> tokens;
 
-    private int position = 0;
     private int nesting = 0;
 
     public TokenBuffer() {
-        this(new ArrayList<>());
-    }
-
-    public TokenBuffer(List<Token> tokens) {
-        this.tokens = tokens;
-        if (tokens.size() > 0)
-            updateNesting(tokens.get(position).token);
+        this.tokens = new ArrayDeque<>();
     }
 
     /** Returns whether any tokens are available in this */
-    public boolean isEmpty() { return remaining() == 0; }
-
-    public JsonToken previous() {
-        updateNestingGoingBackwards(current());
-        position--;
-        return current();
-    }
-
-    /** Returns the current token without changing position, or null if none */
-    public JsonToken current() {
-        if (isEmpty()) return null;
-        Token token = tokens.get(position);
-        if (token == null) return null;
-        return token.token;
-    }
+    public boolean isEmpty() { return tokens.isEmpty(); }
 
+    /** Returns the next token, or null, and updates the nesting count of this. */
     public JsonToken next() {
-        position++;
+        tokens.poll();
         JsonToken token = current();
         updateNesting(token);
         return token;
     }
 
-    /** Returns a given number of tokens ahead, or null if none */
-    public JsonToken peek(int ahead) {
-        if (tokens.size() <= position + ahead) return null;
-        return tokens.get(position + ahead).token;
+    /** Returns the current token without changing position, or null if none */
+    public JsonToken current() {
+        return isEmpty() ? null : tokens.peek().token;
     }
 
     /** Returns the current token name without changing position, or null if none */
     public String currentName() {
-        if (isEmpty()) return null;
-        Token token = tokens.get(position);
-        if (token == null) return null;
-        return token.name;
+        return isEmpty() ? null : tokens.peek().name;
     }
 
     /** Returns the current token text without changing position, or null if none */
     public String currentText() {
-        if (isEmpty()) return null;
-        Token token = tokens.get(position);
-        if (token == null) return null;
-        return token.text;
+        return isEmpty() ? null : tokens.peek().text;
     }
 
-    public int remaining() {
-        return tokens.size() - position;
+    /**
+     * Returns a sequence of remaining tokens in this, or nulls when none remain.
+     * This may fill the token buffer, but not otherwise modify it.
+     */
+    public Supplier<Token> lookahead() {
+        Iterator<Token> iterator = tokens.iterator();
+        if (iterator.hasNext()) iterator.next();
+        return () -> iterator.hasNext() ? iterator.next() : null;
     }
 
     private void add(JsonToken token, String name, String text) {
-        tokens.add(tokens.size(), new Token(token, name, text));
+        tokens.add(new Token(token, name, text));
     }
 
     public void bufferObject(JsonToken first, JsonParser tokens) {
@@ -96,7 +76,7 @@ public class TokenBuffer {
 
         Preconditions.checkArgument(first == firstToken,
                 "Expected %s, got %s.", firstToken.name(), t);
-        if (remaining() == 0) {
+        if (isEmpty()) {
             updateNesting(t);
         }
         localNesting = storeAndPeekNesting(t, localNesting, tokens);
@@ -142,23 +122,14 @@ public class TokenBuffer {
         nesting += nestingOffset(token);
     }
 
-    private void updateNestingGoingBackwards(JsonToken token) {
-        nesting -= nestingOffset(token);
-    }
-
     public int nesting() {
         return nesting;
     }
 
     public void skipToRelativeNesting(int relativeNesting) {
         int initialNesting = nesting();
-        do {
-            next();
-        } while ( nesting() > initialNesting + relativeNesting);
-    }
-
-    public List<Token> rest() {
-        return tokens.subList(position, tokens.size());
+        do next();
+        while (nesting() > initialNesting + relativeNesting);
     }
 
     public static final class Token {
diff --git a/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java
index 2dce07cdbe6..e859306f04d 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java
@@ -8,6 +8,7 @@ import com.yahoo.document.json.TokenBuffer;
 import java.util.Optional;
 
 public class DocumentParseInfo {
+    public DocumentParseInfo() { }
     public DocumentId documentId;
     public Optional<Boolean> create = Optional.empty();
     public Optional<String> condition = Optional.empty();
diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java
index 0b7b1ae9996..27179ef32de 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java
@@ -4,13 +4,15 @@ package com.yahoo.document.json.readers;
 import com.fasterxml.jackson.core.JsonToken;
 import com.yahoo.document.datatypes.TensorFieldValue;
 import com.yahoo.document.json.TokenBuffer;
-import com.yahoo.slime.Inspector;
-import com.yahoo.slime.Type;
+import com.yahoo.document.json.TokenBuffer.Token;
 import com.yahoo.tensor.IndexedTensor;
 import com.yahoo.tensor.MixedTensor;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorAddress;
 import com.yahoo.tensor.TensorType;
+import com.yahoo.tensor.TensorType.Dimension;
+
+import java.util.function.Supplier;
 
 import static com.yahoo.document.json.readers.JsonParserHelpers.*;
 import static com.yahoo.tensor.serialization.JsonFormat.decodeHexString;
@@ -37,36 +39,43 @@ public class TensorReader {
         Tensor.Builder builder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType());
         expectOneOf(buffer.current(), JsonToken.START_OBJECT, JsonToken.START_ARRAY);
         int initNesting = buffer.nesting();
-        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
-            if (TENSOR_CELLS.equals(buffer.currentName()) && ! primitiveContent(buffer)) {
+        while (true) {
+            Supplier<Token> lookahead = buffer.lookahead();
+            Token next = lookahead.get();
+            if (TENSOR_CELLS.equals(next.name) && ! primitiveContent(next.token, lookahead.get().token)) {
+                buffer.next();
                 readTensorCells(buffer, builder);
             }
-            else if (TENSOR_VALUES.equals(buffer.currentName()) && builder.type().dimensions().stream().allMatch(d -> d.isIndexed())) {
+            else if (TENSOR_VALUES.equals(next.name) && builder.type().dimensions().stream().allMatch(Dimension::isIndexed)) {
+                buffer.next();
                 readTensorValues(buffer, builder);
             }
-            else if (TENSOR_BLOCKS.equals(buffer.currentName())) {
+            else if (TENSOR_BLOCKS.equals(next.name)) {
+                buffer.next();
                 readTensorBlocks(buffer, builder);
             }
-            else if (TENSOR_TYPE.equals(buffer.currentName()) && buffer.current() == JsonToken.VALUE_STRING) {
+            else if (TENSOR_TYPE.equals(next.name) && next.token == JsonToken.VALUE_STRING) {
+                buffer.next();
                 // Ignore input tensor type
             }
+            else if (buffer.nesting() == initNesting && JsonToken.END_OBJECT == next.token) {
+                buffer.next();
+                break;
+            }
             else {
-                buffer.previous(); // Back up to the start of the enclosing block
                 readDirectTensorValue(buffer, builder);
-                buffer.previous(); // ... and back up to the end of the enclosing block
+                break;
             }
         }
         expectOneOf(buffer.current(), JsonToken.END_OBJECT, JsonToken.END_ARRAY);
         tensorFieldValue.assign(builder.build());
     }
 
-    static boolean primitiveContent(TokenBuffer buffer) {
-        JsonToken cellsValue = buffer.current();
-        if (cellsValue.isScalarValue()) return true;
-        if (cellsValue == JsonToken.START_ARRAY) {
-            JsonToken firstArrayValue = buffer.peek(1);
-            if (firstArrayValue == JsonToken.END_ARRAY) return false;
-            if (firstArrayValue.isScalarValue()) return true;
+    static boolean primitiveContent(JsonToken current, JsonToken next) {
+        if (current.isScalarValue()) return true;
+        if (current == JsonToken.START_ARRAY) {
+            if (next == JsonToken.END_ARRAY) return false;
+            if (next.isScalarValue()) return true;
         }
         return false;
     }
@@ -186,7 +195,7 @@ public class TensorReader {
         boolean hasIndexed = builder.type().dimensions().stream().anyMatch(TensorType.Dimension::isIndexed);
         boolean hasMapped = builder.type().dimensions().stream().anyMatch(TensorType.Dimension::isMapped);
 
-        if (isArrayOfObjects(buffer, 0))
+        if (isArrayOfObjects(buffer))
             readTensorCells(buffer, builder);
         else if ( ! hasMapped)
             readTensorValues(buffer, builder);
@@ -196,10 +205,12 @@ public class TensorReader {
             readTensorCells(buffer, builder);
     }
 
-    private static boolean isArrayOfObjects(TokenBuffer buffer, int ahead) {
-        if (buffer.peek(ahead++) != JsonToken.START_ARRAY) return false;
-        if (buffer.peek(ahead) == JsonToken.START_ARRAY) return isArrayOfObjects(buffer, ahead); // nested array
-        return buffer.peek(ahead) == JsonToken.START_OBJECT;
+    private static boolean isArrayOfObjects(TokenBuffer buffer) {
+        if (buffer.current() != JsonToken.START_ARRAY) return false;
+        Supplier<Token> lookahead = buffer.lookahead();
+        Token next;
+        while ((next = lookahead.get()).token == JsonToken.START_ARRAY);
+        return next.token == JsonToken.START_OBJECT;
     }
 
     private static TensorAddress readAddress(TokenBuffer buffer, TensorType type) {
diff --git a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java
index 113b8732b23..c7303d31ea2 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java
@@ -238,7 +238,7 @@ public class VespaJsonDocumentReader {
                                  "Expected end of JSON struct (%s), got %s", expectedFinalToken, buffer.current());
         Preconditions.checkState(buffer.nesting() == 0, "Nesting not zero at end of operation");
         Preconditions.checkState(buffer.next() == null, "Dangling data at end of operation");
-        Preconditions.checkState(buffer.remaining() == 0, "Dangling data at end of operation");
+        Preconditions.checkState(buffer.isEmpty(), "Dangling data at end of operation");
     }
 
 }