Remove un-needed tokens

7 files changed, 60 insertions, 49 deletions

diff --git a/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java b/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java
index aa79e2dad44..ce78ec80b67 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java
@@ -40,11 +40,10 @@ public class AddRemoveCreator {
         FieldValue container = field.getDataType().createFieldValue();
         FieldUpdate singleUpdate;
         int initNesting = buffer.nesting();
-        JsonToken token;
 
         Preconditions.checkState(buffer.currentToken().isStructStart(), "Expected start of composite, got %s", buffer.currentToken());
         if (container instanceof CollectionFieldValue) {
-            token = buffer.next();
+            buffer.next();
             DataType valueType = ((CollectionFieldValue) container).getDataType().getNestedType();
             if (container instanceof WeightedSet) {
                 // these are objects with string keys (which are the nested
@@ -59,9 +58,9 @@ public class AddRemoveCreator {
                 }
             } else {
                 List<FieldValue> arrayContents = new ArrayList<>();
-                token = ArrayReader.fillArrayUpdate(buffer, initNesting, token, valueType, arrayContents);
-                if (token != JsonToken.END_ARRAY) {
-                    throw new IllegalStateException("Expected END_ARRAY. Got '" + token + "'.");
+                ArrayReader.fillArrayUpdate(buffer, initNesting, valueType, arrayContents);
+                if (buffer.currentToken() != JsonToken.END_ARRAY) {
+                    throw new IllegalStateException("Expected END_ARRAY. Got '" + buffer.currentToken() + "'.");
                 }
                 if (isRemove) {
                     singleUpdate = FieldUpdate.createRemoveAll(field, arrayContents);
diff --git a/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java
index ab3c3e23816..d51bdac53df 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.document.json.readers;
 
-import com.fasterxml.jackson.core.JsonToken;
 import com.yahoo.document.DataType;
 import com.yahoo.document.datatypes.CollectionFieldValue;
 import com.yahoo.document.datatypes.FieldValue;
@@ -13,23 +12,21 @@ import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart
 import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue;
 
 public class ArrayReader {
-    static public JsonToken fillArrayUpdate(TokenBuffer buffer, int initNesting, JsonToken initToken, DataType valueType, List<FieldValue> arrayContents) {
-        JsonToken token = initToken;
+    public static void fillArrayUpdate(TokenBuffer buffer, int initNesting, DataType valueType, List<FieldValue> arrayContents) {
         while (buffer.nesting() >= initNesting) {
-            arrayContents.add(readSingleValue(buffer, token, valueType));
-            token = buffer.next();
+            arrayContents.add(readSingleValue(buffer, valueType));
+            buffer.next();
         }
-        return token;
     }
 
     @SuppressWarnings({ "unchecked", "rawtypes" })
     public static void fillArray(TokenBuffer buffer, CollectionFieldValue parent, DataType valueType) {
         int initNesting = buffer.nesting();
         expectArrayStart(buffer.currentToken());
-        JsonToken token = buffer.next();
+        buffer.next();
         while (buffer.nesting() >= initNesting) {
-            parent.add(readSingleValue(buffer, token, valueType));
-            token = buffer.next();
+            parent.add(readSingleValue(buffer, valueType));
+            buffer.next();
         }
     }
 }
diff --git a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java
index 712ab81899b..2d016223f88 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java
@@ -20,19 +20,6 @@ public class CompositeReader {
     // TODO createComposite is extremely similar to add/remove, refactor
     // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders
     @SuppressWarnings({ "cast", "rawtypes" })
-    public static FieldValue createComposite(TokenBuffer buffer, DataType expectedType) {
-        FieldValue fieldValue = expectedType.createFieldValue();
-        if (buffer.currentToken().isStructStart()) {
-            populateComposite(buffer, fieldValue);
-        } else {
-            fieldValue.assign(buffer.currentText());
-        }
-        return fieldValue;
-    }
-
-    // TODO createComposite is extremely similar to add/remove, refactor
-    // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders
-    @SuppressWarnings({ "cast", "rawtypes" })
     public static void populateComposite(TokenBuffer buffer, FieldValue fieldValue) {
         JsonToken token = buffer.currentToken();
         if ((token != JsonToken.START_OBJECT) && (token != JsonToken.START_ARRAY)) {
diff --git a/document/src/main/java/com/yahoo/document/json/readers/MapReader.java b/document/src/main/java/com/yahoo/document/json/readers/MapReader.java
index 0897f05e82a..15fc91c274e 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/MapReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/MapReader.java
@@ -20,6 +20,7 @@ import com.yahoo.document.update.ValueUpdate;
 import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart;
 import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectEnd;
 import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart;
+import static com.yahoo.document.json.readers.SingleValueReader.readAtomic;
 import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate;
 import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue;
 
@@ -29,8 +30,16 @@ public class MapReader {
     public static final String UPDATE_ELEMENT = "element";
     public static final String UPDATE_MATCH = "match";
 
-    @SuppressWarnings({ "rawtypes", "cast", "unchecked" })
     public static void fillMap(TokenBuffer buffer, MapFieldValue parent) {
+        if (buffer.currentToken() == JsonToken.START_ARRAY) {
+            MapReader.fillMapFromArray(buffer, parent);
+        } else {
+            MapReader.fillMapFromObject(buffer, parent);
+        }
+    }
+
+        @SuppressWarnings({ "rawtypes", "cast", "unchecked" })
+    public static void fillMapFromArray(TokenBuffer buffer, MapFieldValue parent) {
         JsonToken token = buffer.currentToken();
         int initNesting = buffer.nesting();
         expectArrayStart(token);
@@ -44,9 +53,9 @@ public class MapReader {
             token = buffer.next();
             for (int i = 0; i < 2; ++i) {
                 if (MAP_KEY.equals(buffer.currentName())) {
-                    key = readSingleValue(buffer, token, keyType);
+                    key = readSingleValue(buffer, keyType);
                 } else if (MAP_VALUE.equals(buffer.currentName())) {
-                    value = readSingleValue(buffer, token, valueType);
+                    value = readSingleValue(buffer, valueType);
                 }
                 token = buffer.next();
             }
@@ -58,6 +67,25 @@ public class MapReader {
         }
     }
 
+    @SuppressWarnings({ "rawtypes", "cast", "unchecked" })
+    public static void fillMapFromObject(TokenBuffer buffer, MapFieldValue parent) {
+        JsonToken token = buffer.currentToken();
+        int initNesting = buffer.nesting();
+        expectObjectStart(token);
+        token = buffer.next();
+        DataType keyType = parent.getDataType().getKeyType();
+        DataType valueType = parent.getDataType().getValueType();
+        while (buffer.nesting() >= initNesting) {
+            FieldValue key = readAtomic(buffer.currentName(), keyType);
+            FieldValue value = readSingleValue(buffer, valueType);
+
+            Preconditions.checkState(key != null && value != null, "Missing key or value for map entry.");
+            parent.put(key, value);
+            token = buffer.next();
+        }
+        expectObjectEnd(token);
+    }
+
     @SuppressWarnings({ "rawtypes", "unchecked" })
     public static ValueUpdate createMapUpdate(TokenBuffer buffer, DataType currentLevel, FieldValue keyParent, FieldValue topLevelKey) {
         TokenBuffer.Token element = buffer.prefetchScalar(UPDATE_ELEMENT);
diff --git a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java
index 24ad5f0589a..2f8826354f3 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java
@@ -18,11 +18,13 @@ public class SingleValueReader {
     public static final String UPDATE_MULTIPLY = "multiply";
     public static final String UPDATE_DIVIDE = "divide";
 
-    public static FieldValue readSingleValue(TokenBuffer buffer, JsonToken t, DataType expectedType) {
-        if (t.isScalarValue()) {
-            return readAtomic(buffer, expectedType);
+    public static FieldValue readSingleValue(TokenBuffer buffer, DataType expectedType) {
+        if (buffer.currentToken().isScalarValue()) {
+            return readAtomic(buffer.currentText(), expectedType);
         } else {
-            return CompositeReader.createComposite(buffer, expectedType);
+            FieldValue fieldValue = expectedType.createFieldValue();
+            CompositeReader.populateComposite(buffer, fieldValue);
+            return fieldValue;
         }
     }
 
@@ -34,7 +36,7 @@ public class SingleValueReader {
             case UPDATE_ASSIGN:
                 update = (buffer.currentToken() == JsonToken.VALUE_NULL)
                         ? ValueUpdate.createClear()
-                        : ValueUpdate.createAssign(readSingleValue(buffer, buffer.currentToken(), expectedType));
+                        : ValueUpdate.createAssign(readSingleValue(buffer, expectedType));
                 break;
             // double is silly, but it's what is used internally anyway
             case UPDATE_INCREMENT:
@@ -55,23 +57,22 @@ public class SingleValueReader {
         return update;
     }
 
-    public static FieldValue readAtomic(TokenBuffer buffer, DataType expectedType) {
+    public static FieldValue readAtomic(String field, DataType expectedType) {
         if (expectedType.equals(DataType.RAW)) {
-            return expectedType.createFieldValue(new Base64().decode(buffer.currentText()));
+            return expectedType.createFieldValue(new Base64().decode(field));
         } else if (expectedType.equals(PositionDataType.INSTANCE)) {
-            return PositionDataType.fromString(buffer.currentText());
+            return PositionDataType.fromString(field);
         } else if (expectedType instanceof ReferenceDataType) {
-            return readReferenceFieldValue(buffer, expectedType);
+            return readReferenceFieldValue(field, expectedType);
         } else {
-            return expectedType.createFieldValue(buffer.currentText());
+            return expectedType.createFieldValue(field);
         }
     }
 
-    private static FieldValue readReferenceFieldValue(TokenBuffer buffer, DataType expectedType) {
+    private static FieldValue readReferenceFieldValue(final String refText, DataType expectedType) {
         final FieldValue value = expectedType.createFieldValue();
-        final String refText = buffer.currentText();
         if (!refText.isEmpty()) {
-            value.assign(new DocumentId(buffer.currentText()));
+            value.assign(new DocumentId(refText));
         }
         return value;
     }
diff --git a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java
index 17c9123e1cd..ef3e6911df1 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.document.json.readers;
 
-import com.fasterxml.jackson.core.JsonToken;
 import com.yahoo.document.Field;
 import com.yahoo.document.datatypes.FieldValue;
 import com.yahoo.document.datatypes.StructuredFieldValue;
@@ -14,14 +13,14 @@ public class StructReader {
     public static void fillStruct(TokenBuffer buffer, StructuredFieldValue parent) {
         // do note the order of initializing initNesting and token is relevant for empty docs
         int initNesting = buffer.nesting();
-        JsonToken token = buffer.next();
+        buffer.next();
 
         while (buffer.nesting() >= initNesting) {
             Field f = getField(buffer, parent);
             try {
-                FieldValue v = readSingleValue(buffer, token, f.getDataType());
+                FieldValue v = readSingleValue(buffer, f.getDataType());
                 parent.setFieldValue(f, v);
-                token = buffer.next();
+                buffer.next();
             } catch (IllegalArgumentException e) {
                 throw new JsonReaderException(f, e);
             }
diff --git a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java
index fc17c97a7cf..d51079df482 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java
@@ -229,7 +229,7 @@ public class VespaJsonDocumentReader {
                     if (dt instanceof NumericDataType) {
                         update.setExpression(buffer.currentText());
                     } else {
-                        FieldValue fv = CompositeReader.createComposite(buffer, dt);
+                        FieldValue fv = SingleValueReader.readSingleValue(buffer, dt);
                         update.setNewValue(fv);
                     }
                     break;
@@ -253,7 +253,7 @@ public class VespaJsonDocumentReader {
             switch (buffer.currentName()) {
                 case "items":
                     DataType dt = update.getFieldPath().getResultingDataType();
-                    FieldValue fv = CompositeReader.createComposite(buffer, dt);
+                    FieldValue fv = SingleValueReader.readSingleValue(buffer, dt);
                     update.setNewValues((Array) fv);
                     break;