Revert "Revert "Dybdahl/refactor json reader""

This reverts commit b7c645f96271f4f27415af6757fef18499c63544.

16 files changed, 785 insertions, 596 deletions

diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java
index 42cc5c5b065..8c48be01799 100644
--- a/document/src/main/java/com/yahoo/document/json/JsonReader.java
+++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java
@@ -6,10 +6,6 @@ import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonToken;
 import com.google.common.annotations.Beta;
 import com.google.common.base.Preconditions;
-import com.yahoo.collections.Pair;
-import com.yahoo.document.ArrayDataType;
-import com.yahoo.document.CollectionDataType;
-import com.yahoo.document.DataType;
 import com.yahoo.document.Document;
 import com.yahoo.document.DocumentId;
 import com.yahoo.document.DocumentOperation;
@@ -19,33 +15,26 @@ import com.yahoo.document.DocumentType;
 import com.yahoo.document.DocumentTypeManager;
 import com.yahoo.document.DocumentUpdate;
 import com.yahoo.document.Field;
-import com.yahoo.document.MapDataType;
-import com.yahoo.document.PositionDataType;
-import com.yahoo.document.ReferenceDataType;
 import com.yahoo.document.TestAndSetCondition;
-import com.yahoo.document.WeightedSetDataType;
-import com.yahoo.document.datatypes.CollectionFieldValue;
-import com.yahoo.document.datatypes.FieldValue;
-import com.yahoo.document.datatypes.IntegerFieldValue;
-import com.yahoo.document.datatypes.MapFieldValue;
-import com.yahoo.document.datatypes.StructuredFieldValue;
-import com.yahoo.document.datatypes.TensorFieldValue;
-import com.yahoo.document.datatypes.WeightedSet;
-import com.yahoo.document.json.TokenBuffer.Token;
+import com.yahoo.document.json.document.DocumentParser;
+import com.yahoo.document.json.readers.DocumentParseInfo;
 import com.yahoo.document.update.FieldUpdate;
-import com.yahoo.document.update.MapValueUpdate;
-import com.yahoo.document.update.ValueUpdate;
-import com.yahoo.tensor.MappedTensor;
-import com.yahoo.tensor.Tensor;
-import com.yahoo.tensor.TensorType;
-import org.apache.commons.codec.binary.Base64;
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Optional;
 
+import static com.yahoo.document.json.document.DocumentParser.parseDocumentsFields;
+import static com.yahoo.document.json.readers.AddRemoveCreator.createAdds;
+import static com.yahoo.document.json.readers.AddRemoveCreator.createRemoves;
+import static com.yahoo.document.json.readers.CompositeReader.populateComposite;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectEnd;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart;
+import static com.yahoo.document.json.readers.MapReader.UPDATE_MATCH;
+import static com.yahoo.document.json.readers.MapReader.createMapUpdate;
+import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate;
+
 /**
  * Initialize Vespa documents/updates/removes from an InputStream containing a
  * valid JSON representation of a feed.
@@ -56,50 +45,18 @@ import java.util.Optional;
 @Beta
 public class JsonReader {
 
-    private enum FieldOperation {
-        ADD, REMOVE
+    // Only used for testing.
+    public Optional<DocumentParseInfo> parseDocument() {
+        return DocumentParser.parseDocument(parser);
     }
 
-    static final String MAP_KEY = "key";
-    static final String MAP_VALUE = "value";
-    static final String FIELDS = "fields";
-    static final String REMOVE = "remove";
-    static final String UPDATE_INCREMENT = "increment";
-    static final String UPDATE_DECREMENT = "decrement";
-    static final String UPDATE_MULTIPLY = "multiply";
-    static final String UPDATE_DIVIDE = "divide";
-    static final String TENSOR_DIMENSIONS = "dimensions";
-    static final String TENSOR_CELLS = "cells";
-    static final String TENSOR_ADDRESS = "address";
-    static final String TENSOR_VALUE = "value";
-
-    private static final String UPDATE = "update";
-    private static final String PUT = "put";
-    private static final String ID = "id";
-    private static final String CONDITION = "condition";
-    private static final String CREATE_IF_NON_EXISTENT = "create";
-    private static final String UPDATE_ASSIGN = "assign";
     private static final String UPDATE_REMOVE = "remove";
-    private static final String UPDATE_MATCH = "match";
     private static final String UPDATE_ADD = "add";
-    private static final String UPDATE_ELEMENT = "element";
 
     private final JsonParser parser;
-    private TokenBuffer buffer = new TokenBuffer();
     private final DocumentTypeManager typeManager;
     private ReaderState state = ReaderState.AT_START;
 
-    static class DocumentParseInfo {
-        public DocumentId documentId;
-        public Optional<Boolean> create = Optional.empty();
-        Optional<String> condition = Optional.empty();
-        SupportedOperation operationType = null;
-    }
-
-    enum SupportedOperation {
-        PUT, UPDATE, REMOVE
-    }
-
     enum ReaderState {
         AT_START, READING, END_OF_FEED
     }
@@ -121,11 +78,11 @@ public class JsonReader {
      * @param docIdString document ID.
      * @return the document
      */
-    public DocumentOperation readSingleDocument(SupportedOperation operationType, String docIdString) {
+    public DocumentOperation readSingleDocument(DocumentParser.SupportedOperation operationType, String docIdString) {
         DocumentId docId = new DocumentId(docIdString);
-        DocumentParseInfo documentParseInfo = parseToDocumentsFieldsAndInsertFieldsIntoBuffer(docId);
+        DocumentParseInfo documentParseInfo = parseDocumentsFields(parser, docId);
         documentParseInfo.operationType = operationType;
-        DocumentOperation operation = createDocumentOperation(documentParseInfo);
+        DocumentOperation operation = createDocumentOperation(documentParseInfo.fieldsBuffer, documentParseInfo);
         operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.condition));
         return operation;
     }
@@ -133,7 +90,7 @@ public class JsonReader {
     public DocumentOperation next() {
         switch (state) {
             case AT_START:
-                JsonToken t = nextToken();
+                JsonToken t = nextToken(parser);
                 expectArrayStart(t);
                 state = ReaderState.READING;
                 break;
@@ -143,34 +100,34 @@ public class JsonReader {
                 break;
         }
 
-        Optional<DocumentParseInfo> documentParseInfo = parseDocument();
+        Optional<DocumentParseInfo> documentParseInfo = DocumentParser.parseDocument(parser);
 
         if (! documentParseInfo.isPresent()) {
             state = ReaderState.END_OF_FEED;
             return null;
         }
-        DocumentOperation operation = createDocumentOperation(documentParseInfo.get());
+        DocumentOperation operation = createDocumentOperation(documentParseInfo.get().fieldsBuffer, documentParseInfo.get());
         operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.get().condition));
         return operation;
     }
 
-    private DocumentOperation createDocumentOperation(DocumentParseInfo documentParseInfo) {
+    private DocumentOperation createDocumentOperation(TokenBuffer buffer, DocumentParseInfo documentParseInfo) {
         DocumentType documentType = getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager);
         final DocumentOperation documentOperation;
         try {
             switch (documentParseInfo.operationType) {
                 case PUT:
                     documentOperation = new DocumentPut(new Document(documentType, documentParseInfo.documentId));
-                    readPut((DocumentPut) documentOperation);
-                    verifyEndState();
+                    readPut(buffer, (DocumentPut) documentOperation);
+                    verifyEndState(buffer);
                     break;
                 case REMOVE:
                     documentOperation = new DocumentRemove(documentParseInfo.documentId);
                     break;
                 case UPDATE:
                     documentOperation = new DocumentUpdate(documentType, documentParseInfo.documentId);
-                    readUpdate((DocumentUpdate) documentOperation);
-                    verifyEndState();
+                    readUpdate(buffer, (DocumentUpdate) documentOperation);
+                    verifyEndState(buffer);
                     break;
                 default:
                     throw new IllegalStateException("Implementation out of sync with itself. This is a bug.");
@@ -188,83 +145,34 @@ public class JsonReader {
         return documentOperation;
     }
 
-    void readUpdate(DocumentUpdate next) {
+    // Exposed for unit testing...
+    void readUpdate(TokenBuffer buffer, DocumentUpdate next) {
         if (buffer.size() == 0) {
-            bufferFields(nextToken());
+            buffer.bufferObject(nextToken(parser), parser);
         }
-        populateUpdateFromBuffer(next);
+        populateUpdateFromBuffer(buffer, next);
     }
 
-    void readPut(DocumentPut put) {
+    // Exposed for unit testing...
+    void readPut(TokenBuffer buffer, DocumentPut put) {
         if (buffer.size() == 0) {
-            bufferFields(nextToken());
+            buffer.bufferObject(nextToken(parser), parser);
         }
-        JsonToken t = buffer.currentToken();
         try {
-            populateComposite(put.getDocument(), t);
+            populateComposite(buffer, put.getDocument());
         } catch (JsonReaderException e) {
             throw JsonReaderException.addDocId(e, put.getId());
         }
     }
 
-    private DocumentParseInfo parseToDocumentsFieldsAndInsertFieldsIntoBuffer(DocumentId documentId) {
-        long indentLevel = 0;
-        DocumentParseInfo documentParseInfo = new DocumentParseInfo();
-        documentParseInfo.documentId = documentId;
-        while (true) {
-            // we should now be at the start of a feed operation or at the end of the feed
-            JsonToken t = nextToken();
-            if (t == null) {
-                throw new IllegalArgumentException("Could not read document, no document?");
-            }
-            switch (t) {
-                case START_OBJECT:
-                    indentLevel++;
-                    break;
-                case END_OBJECT:
-                    indentLevel--;
-                    break;
-                case START_ARRAY:
-                    indentLevel+=10000L;
-                    break;
-                case END_ARRAY:
-                    indentLevel-=10000L;
-                    break;
-            }
-            if (indentLevel == 1 && (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE)) {
-                try {
-                    if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) {
-                        documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue());
-                        continue;
-                    }
-                } catch (IOException e) {
-                    throw new RuntimeException("Got IO exception while parsing document", e);
-                }
-            }
-            if (indentLevel == 2L && t == JsonToken.START_OBJECT) {
-
-                try {
-                    if (!FIELDS.equals(parser.getCurrentName())) {
-                        continue;
-                    }
-                } catch (IOException e) {
-                    throw new RuntimeException("Got IO exception while parsing document", e);
-                }
-                bufferFields(t);
-                break;
-            }
-        }
-        return documentParseInfo;
-    }
-
-    private void verifyEndState() {
+    private void verifyEndState(TokenBuffer buffer) {
         Preconditions.checkState(buffer.nesting() == 0, "Nesting not zero at end of operation");
         expectObjectEnd(buffer.currentToken());
         Preconditions.checkState(buffer.next() == null, "Dangling data at end of operation");
         Preconditions.checkState(buffer.size() == 0, "Dangling data at end of operation");
     }
 
-    private void populateUpdateFromBuffer(DocumentUpdate update) {
+    private static void populateUpdateFromBuffer(TokenBuffer buffer, DocumentUpdate update) {
         expectObjectStart(buffer.currentToken());
         int localNesting = buffer.nesting();
         JsonToken t = buffer.next();
@@ -273,12 +181,12 @@ public class JsonReader {
             expectObjectStart(t);
             String fieldName = buffer.currentName();
             Field field = update.getType().getField(fieldName);
-            addFieldUpdates(update, field);
+            addFieldUpdates(buffer, update, field);
             t = buffer.next();
         }
     }
 
-    private void addFieldUpdates(DocumentUpdate update, Field field) {
+    private static void addFieldUpdates(TokenBuffer buffer, DocumentUpdate update, Field field) {
         int localNesting = buffer.nesting();
         FieldUpdate fieldUpdate = FieldUpdate.create(field);
 
@@ -286,457 +194,24 @@ public class JsonReader {
         while (localNesting <= buffer.nesting()) {
             switch (buffer.currentName()) {
             case UPDATE_REMOVE:
-                createAddsOrRemoves(field, fieldUpdate, FieldOperation.REMOVE);
+                createRemoves(buffer, field, fieldUpdate);
                 break;
             case UPDATE_ADD:
-                createAddsOrRemoves(field, fieldUpdate, FieldOperation.ADD);
+                createAdds(buffer, field, fieldUpdate);
                 break;
             case UPDATE_MATCH:
-                fieldUpdate.addValueUpdate(createMapUpdate(field));
+                fieldUpdate.addValueUpdate(createMapUpdate(buffer, field));
                 break;
             default:
                 String action = buffer.currentName();
-                fieldUpdate.addValueUpdate(readSingleUpdate(field.getDataType(), action));
+                fieldUpdate.addValueUpdate(readSingleUpdate(buffer, field.getDataType(), action));
             }
             buffer.next();
         }
         update.addFieldUpdate(fieldUpdate);
     }
 
-    @SuppressWarnings("rawtypes")
-    private ValueUpdate createMapUpdate(Field field) {
-        buffer.next();
-        MapValueUpdate m = (MapValueUpdate) createMapUpdate(field.getDataType(), null, null);
-        buffer.next();
-        // must generate the field value in parallell with the actual
-        return m;
-
-    }
-
-    @SuppressWarnings({ "rawtypes", "unchecked" })
-    private ValueUpdate createMapUpdate(DataType currentLevel, FieldValue keyParent, FieldValue topLevelKey) {
-        TokenBuffer.Token element = buffer.prefetchScalar(UPDATE_ELEMENT);
-        if (UPDATE_ELEMENT.equals(buffer.currentName())) {
-            buffer.next();
-        }
-
-        FieldValue key = keyTypeForMapUpdate(element, currentLevel);
-        if (keyParent != null) {
-            ((CollectionFieldValue) keyParent).add(key);
-        }
-        // structure is: [(match + element)*, (element + action)]
-        // match will always have element, and either match or action
-        if (!UPDATE_MATCH.equals(buffer.currentName())) {
-            // we have reached an action...
-            if (topLevelKey == null) {
-                return ValueUpdate.createMap(key, readSingleUpdate(valueTypeForMapUpdate(currentLevel), buffer.currentName()));
-            } else {
-                return ValueUpdate.createMap(topLevelKey, readSingleUpdate(valueTypeForMapUpdate(currentLevel), buffer.currentName()));
-            }
-        } else {
-            // next level of matching
-            if (topLevelKey == null) {
-                return createMapUpdate(valueTypeForMapUpdate(currentLevel), key, key);
-            } else {
-                return createMapUpdate(valueTypeForMapUpdate(currentLevel), key, topLevelKey);
-            }
-        }
-    }
-
-    private DataType valueTypeForMapUpdate(DataType parentType) {
-        if (parentType instanceof WeightedSetDataType) {
-            return DataType.INT;
-        } else if (parentType instanceof CollectionDataType) {
-            return ((CollectionDataType) parentType).getNestedType();
-        } else if (parentType instanceof MapDataType) {
-            return ((MapDataType) parentType).getValueType();
-        } else {
-            throw new UnsupportedOperationException("Unexpected parent type: " + parentType);
-        }
-    }
-
-    private FieldValue keyTypeForMapUpdate(Token element, DataType expectedType) {
-        FieldValue v;
-        if (expectedType instanceof ArrayDataType) {
-            v = new IntegerFieldValue(Integer.valueOf(element.text));
-        } else if (expectedType instanceof WeightedSetDataType) {
-            v = ((WeightedSetDataType) expectedType).getNestedType().createFieldValue(element.text);
-        } else if (expectedType instanceof MapDataType) {
-            v = ((MapDataType) expectedType).getKeyType().createFieldValue(element.text);
-        } else {
-            throw new IllegalArgumentException("Container type " + expectedType + " not supported for match update.");
-        }
-        return v;
-    }
-
-    @SuppressWarnings("rawtypes")
-    private ValueUpdate readSingleUpdate(DataType expectedType, String action) {
-        ValueUpdate update;
-
-        switch (action) {
-            case UPDATE_ASSIGN:
-                update = (buffer.currentToken() == JsonToken.VALUE_NULL)
-                        ? ValueUpdate.createClear()
-                        : ValueUpdate.createAssign(readSingleValue(buffer.currentToken(), expectedType));
-                break;
-                // double is silly, but it's what is used internally anyway
-            case UPDATE_INCREMENT:
-                update = ValueUpdate.createIncrement(Double.valueOf(buffer.currentText()));
-                break;
-            case UPDATE_DECREMENT:
-                update = ValueUpdate.createDecrement(Double.valueOf(buffer.currentText()));
-                break;
-            case UPDATE_MULTIPLY:
-                update = ValueUpdate.createMultiply(Double.valueOf(buffer.currentText()));
-                break;
-            case UPDATE_DIVIDE:
-                update = ValueUpdate.createDivide(Double.valueOf(buffer.currentText()));
-                break;
-            default:
-                throw new IllegalArgumentException("Operation \"" + buffer.currentName() + "\" not implemented.");
-        }
-        return update;
-    }
-
-    // yes, this suppresswarnings ugliness is by intention, the code relies on
-    // the contracts in the builders
-    @SuppressWarnings({ "cast", "rawtypes", "unchecked" })
-    private void createAddsOrRemoves(Field field, FieldUpdate update, FieldOperation op) {
-        FieldValue container = field.getDataType().createFieldValue();
-        FieldUpdate singleUpdate;
-        int initNesting = buffer.nesting();
-        JsonToken token;
-
-        Preconditions.checkState(buffer.currentToken().isStructStart(), "Expected start of composite, got %s", buffer.currentToken());
-        if (container instanceof CollectionFieldValue) {
-            token = buffer.next();
-            DataType valueType = ((CollectionFieldValue) container).getDataType().getNestedType();
-            if (container instanceof WeightedSet) {
-                // these are objects with string keys (which are the nested
-                // types) and values which are the weight
-                WeightedSet weightedSet = (WeightedSet) container;
-                fillWeightedSetUpdate(initNesting, valueType, weightedSet);
-                if (op == FieldOperation.REMOVE) {
-                    singleUpdate = FieldUpdate.createRemoveAll(field, weightedSet);
-                } else {
-                    singleUpdate = FieldUpdate.createAddAll(field, weightedSet);
-
-                }
-            } else {
-                List<FieldValue> arrayContents = new ArrayList<>();
-                token = fillArrayUpdate(initNesting, token, valueType, arrayContents);
-                if (token != JsonToken.END_ARRAY) {
-                    throw new IllegalStateException("Expected END_ARRAY. Got '" + token + "'.");
-                }
-                if (op == FieldOperation.REMOVE) {
-                    singleUpdate = FieldUpdate.createRemoveAll(field, arrayContents);
-                } else {
-                    singleUpdate = FieldUpdate.createAddAll(field, arrayContents);
-                }
-            }
-        } else {
-            throw new UnsupportedOperationException(
-                    "Trying to add or remove from a field of a type the reader does not know how to handle: "
-                            + container.getClass().getName());
-        }
-        expectCompositeEnd(buffer.currentToken());
-        update.addAll(singleUpdate);
-    }
-
-    private JsonToken fillArrayUpdate(int initNesting, JsonToken initToken, DataType valueType, List<FieldValue> arrayContents) {
-        JsonToken token = initToken;
-        while (buffer.nesting() >= initNesting) {
-            arrayContents.add(readSingleValue(token, valueType));
-            token = buffer.next();
-        }
-        return token;
-    }
-
-    private void fillWeightedSetUpdate(int initNesting, DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) {
-        iterateThroughWeightedSet(initNesting, valueType, weightedSet);
-    }
-
-    @SuppressWarnings({ "rawtypes", "unchecked" })
-    private void iterateThroughWeightedSet(int initNesting, DataType valueType, WeightedSet weightedSet) {
-        while (buffer.nesting() >= initNesting) {
-            // XXX the keys are defined in the spec to always be represented as strings
-            FieldValue v = valueType.createFieldValue(buffer.currentName());
-            weightedSet.put(v, Integer.valueOf(buffer.currentText()));
-            buffer.next();
-        }
-    }
-
-    // TODO populateComposite is extremely similar to add/remove, refactor
-    // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders
-    @SuppressWarnings({ "cast", "rawtypes" })
-    private void populateComposite(FieldValue parent, JsonToken token) {
-        if ((token != JsonToken.START_OBJECT) && (token != JsonToken.START_ARRAY)) {
-            throw new IllegalArgumentException("Expected '[' or '{'. Got '" + token + "'.");
-        }
-        if (parent instanceof CollectionFieldValue) {
-            DataType valueType = ((CollectionFieldValue) parent).getDataType().getNestedType();
-            if (parent instanceof WeightedSet) {
-                fillWeightedSet(valueType, (WeightedSet) parent);
-            } else {
-                fillArray((CollectionFieldValue) parent, valueType);
-            }
-        } else if (parent instanceof MapFieldValue) {
-            fillMap((MapFieldValue) parent);
-        } else if (parent instanceof StructuredFieldValue) {
-            fillStruct((StructuredFieldValue) parent);
-        } else if (parent instanceof TensorFieldValue) {
-            fillTensor((TensorFieldValue) parent);
-        } else {
-            throw new IllegalStateException("Has created a composite field"
-                    + " value the reader does not know how to handle: "
-                    + parent.getClass().getName() + " This is a bug. token = " + token);
-        }
-        expectCompositeEnd(buffer.currentToken());
-    }
-
-    private void expectCompositeEnd(JsonToken token) {
-        Preconditions.checkState(token.isStructEnd(), "Expected end of composite, got %s", token);
-    }
-
-    private void fillStruct(StructuredFieldValue parent) {
-        // do note the order of initializing initNesting and token is relevant for empty docs
-        int initNesting = buffer.nesting();
-        JsonToken token = buffer.next();
-
-        while (buffer.nesting() >= initNesting) {
-            Field f = getField(parent);
-            try {
-                FieldValue v = readSingleValue(token, f.getDataType());
-                parent.setFieldValue(f, v);
-                token = buffer.next();
-            } catch (IllegalArgumentException e) {
-                throw new JsonReaderException(f, e);
-            }
-        }
-    }
-
-    private Field getField(StructuredFieldValue parent) {
-        Field f = parent.getField(buffer.currentName());
-        if (f == null) {
-            throw new NullPointerException("Could not get field \"" + buffer.currentName() +
-                    "\" in the structure of type \"" + parent.getDataType().getDataTypeName() + "\".");
-        }
-        return f;
-    }
-
-    @SuppressWarnings({ "rawtypes", "cast", "unchecked" })
-    private void fillMap(MapFieldValue parent) {
-        JsonToken token = buffer.currentToken();
-        int initNesting = buffer.nesting();
-        expectArrayStart(token);
-        token = buffer.next();
-        DataType keyType = parent.getDataType().getKeyType();
-        DataType valueType = parent.getDataType().getValueType();
-        while (buffer.nesting() >= initNesting) {
-            FieldValue key = null;
-            FieldValue value = null;
-            expectObjectStart(token);
-            token = buffer.next();
-            for (int i = 0; i < 2; ++i) {
-                if (MAP_KEY.equals(buffer.currentName())) {
-                    key = readSingleValue(token, keyType);
-                } else if (MAP_VALUE.equals(buffer.currentName())) {
-                    value = readSingleValue(token, valueType);
-                }
-                token = buffer.next();
-            }
-            Preconditions.checkState(key != null && value != null, "Missing key or value for map entry.");
-            parent.put(key, value);
-
-            expectObjectEnd(token);
-            token = buffer.next(); // array end or next entry
-        }
-    }
-
-    private void expectArrayStart(JsonToken token) {
-        Preconditions.checkState(token == JsonToken.START_ARRAY, "Expected start of array, got %s", token);
-    }
-
-    private void expectObjectStart(JsonToken token) {
-        Preconditions.checkState(token == JsonToken.START_OBJECT, "Expected start of JSON object, got %s", token);
-    }
-
-    private void expectObjectEnd(JsonToken token) {
-        Preconditions.checkState(token == JsonToken.END_OBJECT, "Expected end of JSON object, got %s", token);
-    }
-
-    @SuppressWarnings({ "unchecked", "rawtypes" })
-    private void fillArray(CollectionFieldValue parent, DataType valueType) {
-        int initNesting = buffer.nesting();
-        expectArrayStart(buffer.currentToken());
-        JsonToken token = buffer.next();
-        while (buffer.nesting() >= initNesting) {
-            parent.add(readSingleValue(token, valueType));
-            token = buffer.next();
-        }
-    }
-
-    private void fillWeightedSet(DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) {
-        int initNesting = buffer.nesting();
-        expectObjectStart(buffer.currentToken());
-        buffer.next();
-        iterateThroughWeightedSet(initNesting, valueType, weightedSet);
-    }
-
-    private void fillTensor(TensorFieldValue tensorFieldValue) {
-        Tensor.Builder tensorBuilder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType());
-        expectObjectStart(buffer.currentToken());
-        int initNesting = buffer.nesting();
-        // read tensor cell fields and ignore everything else
-        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
-            if (TENSOR_CELLS.equals(buffer.currentName()))
-                readTensorCells(tensorBuilder);
-        }
-        expectObjectEnd(buffer.currentToken());
-        tensorFieldValue.assign(tensorBuilder.build());
-    }
-
-    private void readTensorCells(Tensor.Builder tensorBuilder) {
-        expectArrayStart(buffer.currentToken());
-        int initNesting = buffer.nesting();
-        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next())
-            readTensorCell(tensorBuilder);
-        expectCompositeEnd(buffer.currentToken());
-    }
-
-    private void readTensorCell(Tensor.Builder tensorBuilder) {
-        expectObjectStart(buffer.currentToken());
-        int initNesting = buffer.nesting();
-        double cellValue = 0.0;
-        Tensor.Builder.CellBuilder cellBuilder = tensorBuilder.cell();
-        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
-            String currentName = buffer.currentName();
-            if (TENSOR_ADDRESS.equals(currentName)) {
-                readTensorAddress(cellBuilder);
-            } else if (TENSOR_VALUE.equals(currentName)) {
-                cellValue = Double.valueOf(buffer.currentText());
-            }
-        }
-        expectObjectEnd(buffer.currentToken());
-        cellBuilder.value(cellValue);
-    }
-
-    private void readTensorAddress(MappedTensor.Builder.CellBuilder cellBuilder) {
-        expectObjectStart(buffer.currentToken());
-        int initNesting = buffer.nesting();
-        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
-            String dimension = buffer.currentName();
-            String label = buffer.currentText();
-            cellBuilder.label(dimension, label);
-        }
-        expectObjectEnd(buffer.currentToken());
-    }
-
-    private FieldValue readSingleValue(JsonToken t, DataType expectedType) {
-        if (t.isScalarValue()) {
-            return readAtomic(expectedType);
-        } else {
-            FieldValue v = expectedType.createFieldValue();
-            populateComposite(v, t);
-            return v;
-        }
-    }
-
-    private FieldValue readAtomic(DataType expectedType) {
-        if (expectedType.equals(DataType.RAW)) {
-            return expectedType.createFieldValue(new Base64().decode(buffer.currentText()));
-        } else if (expectedType.equals(PositionDataType.INSTANCE)) {
-            return PositionDataType.fromString(buffer.currentText());
-        } else if (expectedType instanceof ReferenceDataType) {
-            return readReferenceFieldValue(expectedType);
-        } else {
-            return expectedType.createFieldValue(buffer.currentText());
-        }
-    }
-
-    private FieldValue readReferenceFieldValue(DataType expectedType) {
-        final FieldValue value = expectedType.createFieldValue();
-        final String refText = buffer.currentText();
-        if (!refText.isEmpty()) {
-            value.assign(new DocumentId(buffer.currentText()));
-        }
-        return value;
-    }
-
-    private void bufferFields(JsonToken current) {
-        buffer.bufferObject(current, parser);
-    }
-
-    Optional<DocumentParseInfo> parseDocument() {
-        // we should now be at the start of a feed operation or at the end of the feed
-        JsonToken token = nextToken();
-        if (token == JsonToken.END_ARRAY) {
-            return Optional.empty(); // end of feed
-        }
-        expectObjectStart(token);
-
-        DocumentParseInfo documentParseInfo = new DocumentParseInfo();
-
-        while (true) {
-            try {
-                token = nextToken();
-                if ((token == JsonToken.VALUE_TRUE || token == JsonToken.VALUE_FALSE) &&
-                     CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) {
-                    documentParseInfo.create = Optional.of(token == JsonToken.VALUE_TRUE);
-                    continue;
-                }
-                if (token == JsonToken.VALUE_STRING && CONDITION.equals(parser.getCurrentName())) {
-                    documentParseInfo.condition = Optional.of(parser.getText());
-                    continue;
-                }
-                if (token == JsonToken.START_OBJECT) {
-                    try {
-                        if (!FIELDS.equals(parser.getCurrentName())) {
-                            throw new IllegalArgumentException("Unexpected object key: " + parser.getCurrentName());
-                        }
-                    } catch (IOException e) {
-                        // TODO more specific wrapping
-                        throw new RuntimeException(e);
-                    }
-                    bufferFields(token);
-                    continue;
-                }
-                if (token == JsonToken.END_OBJECT) {
-                    if (documentParseInfo.documentId == null) {
-                        throw new RuntimeException("Did not find document operation");
-                    }
-                    return Optional.of(documentParseInfo);
-                }
-                if (token == JsonToken.VALUE_STRING) {
-                    documentParseInfo.operationType = operationNameToOperationType(parser.getCurrentName());
-                    documentParseInfo.documentId = new DocumentId(parser.getText());
-                    continue;
-                }
-                throw new RuntimeException("Expected document start or document operation.");
-            } catch (IOException e) {
-                throw new IllegalStateException(e);
-            }
-
-        }
-    }
-
-    private static SupportedOperation operationNameToOperationType(String operationName) {
-        switch (operationName) {
-            case PUT:
-            case ID:
-                return SupportedOperation.PUT;
-            case REMOVE:
-                return SupportedOperation.REMOVE;
-            case UPDATE:
-                return SupportedOperation.UPDATE;
-            default:
-                throw new IllegalArgumentException(
-                        "Got " + operationName + " as document operation, only \"put\", " +
-                                "\"remove\" and \"update\" are supported.");
-        }
-    }
-
-    DocumentType readDocumentType(DocumentId docId) {
+    public DocumentType readDocumentType(DocumentId docId) {
         return getDocumentTypeFromString(docId.getDocType(), typeManager);
     }
 
@@ -748,12 +223,13 @@ public class JsonReader {
         return docType;
     }
 
-    private JsonToken nextToken() {
+    public static JsonToken nextToken(JsonParser parser) {
         try {
             return parser.nextValue();
         } catch (IOException e) {
             // Jackson is not able to recover from structural parse errors
-            state = ReaderState.END_OF_FEED;
+            // TODO Do we really need to set state on exception?
+            // state = ReaderState.END_OF_FEED;
             throw new RuntimeException(e);
         }
     }
diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java
index 5fb8948649a..855fabc7a63 100644
--- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java
+++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java
@@ -21,6 +21,7 @@ import com.yahoo.document.datatypes.Struct;
 import com.yahoo.document.datatypes.StructuredFieldValue;
 import com.yahoo.document.datatypes.TensorFieldValue;
 import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.json.readers.TensorReader;
 import com.yahoo.document.serialization.FieldWriter;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorAddress;
@@ -34,6 +35,9 @@ import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 
+import static com.yahoo.document.json.readers.MapReader.MAP_KEY;
+import static com.yahoo.document.json.readers.MapReader.MAP_VALUE;
+
 /**
  * @author Steinar Knutsen
  * @author Vegard Sjonfjell
@@ -78,7 +82,7 @@ public class JsonSerializationHelper {
     }
 
     private static void serializeTensorDimensions(JsonGenerator generator, Set<String> dimensions) throws IOException {
-        generator.writeArrayFieldStart(JsonReader.TENSOR_DIMENSIONS);
+        generator.writeArrayFieldStart(TensorReader.TENSOR_DIMENSIONS);
         for (String dimension : dimensions) {
             generator.writeString(dimension);
         }
@@ -87,11 +91,11 @@ public class JsonSerializationHelper {
     }
 
     private static void serializeTensorCells(JsonGenerator generator, Tensor tensor) throws IOException {
-        generator.writeArrayFieldStart(JsonReader.TENSOR_CELLS);
+        generator.writeArrayFieldStart(TensorReader.TENSOR_CELLS);
         for (Map.Entry<TensorAddress, Double> cell : tensor.cells().entrySet()) {
             generator.writeStartObject();
             serializeTensorAddress(generator, cell.getKey(), tensor.type());
-            generator.writeNumberField(JsonReader.TENSOR_VALUE, cell.getValue());
+            generator.writeNumberField(TensorReader.TENSOR_VALUE, cell.getValue());
             generator.writeEndObject();
         }
 
@@ -99,7 +103,7 @@ public class JsonSerializationHelper {
     }
 
     private static void serializeTensorAddress(JsonGenerator generator, TensorAddress address, TensorType type) throws IOException {
-        generator.writeObjectFieldStart(JsonReader.TENSOR_ADDRESS);
+        generator.writeObjectFieldStart(TensorReader.TENSOR_ADDRESS);
         for (int i = 0; i < type.dimensions().size(); i++)
             generator.writeStringField(type.dimensions().get(i).name(), address.label(i));
 
@@ -186,9 +190,9 @@ public class JsonSerializationHelper {
 
             for (Map.Entry<K, V> entry : map.entrySet()) {
                 generator.writeStartObject();
-                generator.writeFieldName(JsonReader.MAP_KEY);
+                generator.writeFieldName(MAP_KEY);
                 entry.getKey().serialize(null, fieldWriter);
-                generator.writeFieldName(JsonReader.MAP_VALUE);
+                generator.writeFieldName(MAP_VALUE);
                 entry.getValue().serialize(null, fieldWriter);
                 generator.writeEndObject();
             }
diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java
index 6ad526a2aaa..bc90141747f 100644
--- a/document/src/main/java/com/yahoo/document/json/JsonWriter.java
+++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java
@@ -38,6 +38,8 @@ import java.util.Iterator;
 import java.util.Map;
 
 import static com.yahoo.document.json.JsonSerializationHelper.*;
+import static com.yahoo.document.json.document.DocumentParser.FIELDS;
+import static com.yahoo.document.json.document.DocumentParser.REMOVE;
 
 /**
  * Serialize Document and other FieldValue instances as JSON.
@@ -106,7 +108,7 @@ public class JsonWriter implements DocumentWriter {
             // this makes it impossible to refeed directly, not sure what's correct
             // perhaps just change to "put"?
             generator.writeStringField("id", value.getId().toString());
-            generator.writeObjectFieldStart(JsonReader.FIELDS);
+            generator.writeObjectFieldStart(FIELDS);
 
             Iterator<Map.Entry<Field, FieldValue>> i = value.iterator();
             while (i.hasNext()) {
@@ -251,7 +253,7 @@ public class JsonWriter implements DocumentWriter {
         try {
             JsonGenerator throwAway = jsonFactory.createGenerator(out);
             throwAway.writeStartObject();
-            throwAway.writeStringField(JsonReader.REMOVE, docId.toString());
+            throwAway.writeStringField(REMOVE, docId.toString());
             throwAway.writeEndObject();
             throwAway.close();
         } catch (IOException e) {
diff --git a/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java
index 4476acfae22..0ca5ba744f1 100644
--- a/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java
+++ b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java
@@ -6,6 +6,7 @@ import com.yahoo.document.DocumentOperation;
 import com.yahoo.document.DocumentPut;
 import com.yahoo.document.DocumentTypeManager;
 import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.json.document.DocumentParser;
 import com.yahoo.vespaxmlparser.VespaXMLFeedReader;
 
 import java.io.IOException;
@@ -25,14 +26,14 @@ public class SingleDocumentParser {
     }
 
     public VespaXMLFeedReader.Operation parsePut(InputStream inputStream, String docId) {
-        return parse(inputStream, docId, JsonReader.SupportedOperation.PUT);
+        return parse(inputStream, docId, DocumentParser.SupportedOperation.PUT);
     }
 
     public VespaXMLFeedReader.Operation parseUpdate(InputStream inputStream, String docId)  {
-        return parse(inputStream, docId, JsonReader.SupportedOperation.UPDATE);
+        return parse(inputStream, docId, DocumentParser.SupportedOperation.UPDATE);
     }
 
-    private VespaXMLFeedReader.Operation parse(InputStream inputStream, String docId, JsonReader.SupportedOperation supportedOperation)  {
+    private VespaXMLFeedReader.Operation parse(InputStream inputStream, String docId, DocumentParser.SupportedOperation supportedOperation)  {
         final JsonReader reader = new JsonReader(docMan, inputStream, jsonFactory);
         final DocumentOperation documentOperation = reader.readSingleDocument(supportedOperation, docId);
         VespaXMLFeedReader.Operation operation = new VespaXMLFeedReader.Operation();
@@ -41,7 +42,7 @@ public class SingleDocumentParser {
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
-        if (supportedOperation == JsonReader.SupportedOperation.PUT) {
+        if (supportedOperation == DocumentParser.SupportedOperation.PUT) {
             operation.setDocument(((DocumentPut) documentOperation).getDocument());
         } else {
             operation.setDocumentUpdate((DocumentUpdate) documentOperation);
diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java
index 8f3395b989e..9f6a0ba3641 100644
--- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java
+++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java
@@ -15,11 +15,11 @@ import com.google.common.base.Preconditions;
  *
  * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
  */
-class TokenBuffer {
-    static final class Token {
-        final JsonToken token;
-        final String name;
-        final String text;
+public class TokenBuffer {
+    public static final class Token {
+        public final JsonToken token;
+        public final String name;
+        public final String text;
 
         Token(JsonToken token, String name, String text) {
             this.token = token;
@@ -31,7 +31,7 @@ class TokenBuffer {
     private Deque<Token> buffer;
     private int nesting = 0;
 
-    TokenBuffer() {
+    public TokenBuffer() {
         this(new ArrayDeque<>());
     }
 
@@ -42,7 +42,7 @@ class TokenBuffer {
         }
     }
 
-    JsonToken next() {
+    public JsonToken next() {
         buffer.removeFirst();
         Token t = buffer.peekFirst();
         if (t == null) {
@@ -52,15 +52,15 @@ class TokenBuffer {
         return t.token;
     }
 
-    JsonToken currentToken() {
+    public JsonToken currentToken() {
         return buffer.peekFirst().token;
     }
 
-    String currentName() {
+    public String currentName() {
         return buffer.peekFirst().name;
     }
 
-    String currentText() {
+    public String currentText() {
         return buffer.peekFirst().text;
     }
 
@@ -72,7 +72,7 @@ class TokenBuffer {
         buffer.addLast(new Token(token, name, text));
     }
 
-    void bufferObject(JsonToken first, JsonParser tokens) {
+    public void  bufferObject(JsonToken first, JsonParser tokens) {
         int localNesting = 0;
         JsonToken t = first;
 
@@ -146,7 +146,7 @@ class TokenBuffer {
         }
     }
 
-    TokenBuffer prefetchCurrentElement() {
+    public TokenBuffer prefetchCurrentElement() {
         Deque<Token> copy = new ArrayDeque<>();
 
         if (currentToken().isScalarValue()) {
@@ -165,7 +165,7 @@ class TokenBuffer {
         return new TokenBuffer(copy);
     }
 
-    Token prefetchScalar(String name) {
+    public Token prefetchScalar(String name) {
         int localNesting = nesting();
         int nestingBarrier = localNesting;
         Token toReturn = null;
diff --git a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java
new file mode 100644
index 00000000000..e464100a59d
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java
@@ -0,0 +1,151 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json.document;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.json.readers.DocumentParseInfo;
+
+import java.io.IOException;
+import java.util.Optional;
+
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart;
+
+public class DocumentParser {
+    public enum SupportedOperation {
+        PUT, UPDATE, REMOVE
+    }
+    private static final String UPDATE = "update";
+    private static final String PUT = "put";
+    private static final String ID = "id";
+    private static final String CONDITION = "condition";
+    public static final String CREATE_IF_NON_EXISTENT = "create";
+    public static final String FIELDS = "fields";
+    public static final String REMOVE = "remove";
+
+    public static Optional<DocumentParseInfo> parseDocument(JsonParser parser) {
+        // we should now be at the start of a feed operation or at the end of the feed
+        JsonToken token = nextToken(parser);
+        if (token == JsonToken.END_ARRAY) {
+            return Optional.empty(); // end of feed
+        }
+        expectObjectStart(token);
+
+        DocumentParseInfo documentParseInfo = new DocumentParseInfo();
+
+        while (true) {
+            try {
+                token = nextToken(parser);
+                if ((token == JsonToken.VALUE_TRUE || token == JsonToken.VALUE_FALSE) &&
+                        CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) {
+                    documentParseInfo.create = Optional.of(token == JsonToken.VALUE_TRUE);
+                    continue;
+                }
+                if (token == JsonToken.VALUE_STRING && CONDITION.equals(parser.getCurrentName())) {
+                    documentParseInfo.condition = Optional.of(parser.getText());
+                    continue;
+                }
+                if (token == JsonToken.START_OBJECT) {
+                    try {
+                        if (!FIELDS.equals(parser.getCurrentName())) {
+                            throw new IllegalArgumentException("Unexpected object key: " + parser.getCurrentName());
+                        }
+                    } catch (IOException e) {
+                        // TODO more specific wrapping
+                        throw new RuntimeException(e);
+                    }
+                    documentParseInfo.fieldsBuffer.bufferObject(token, parser);
+                    continue;
+                }
+                if (token == JsonToken.END_OBJECT) {
+                    if (documentParseInfo.documentId == null) {
+                        throw new RuntimeException("Did not find document operation");
+                    }
+                    return Optional.of(documentParseInfo);
+                }
+                if (token == JsonToken.VALUE_STRING) {
+                    documentParseInfo.operationType = operationNameToOperationType(parser.getCurrentName());
+                    documentParseInfo.documentId = new DocumentId(parser.getText());
+                    continue;
+                }
+                throw new RuntimeException("Expected document start or document operation.");
+            } catch (IOException e) {
+                throw new IllegalStateException(e);
+            }
+        }
+    }
+
+    private static SupportedOperation operationNameToOperationType(String operationName) {
+        switch (operationName) {
+            case PUT:
+            case ID:
+                return SupportedOperation.PUT;
+            case REMOVE:
+                return SupportedOperation.REMOVE;
+            case UPDATE:
+                return SupportedOperation.UPDATE;
+            default:
+                throw new IllegalArgumentException(
+                        "Got " + operationName + " as document operation, only \"put\", " +
+                                "\"remove\" and \"update\" are supported.");
+        }
+    }
+
+    public static DocumentParseInfo parseDocumentsFields(JsonParser parser, DocumentId documentId) {
+        long indentLevel = 0;
+        DocumentParseInfo documentParseInfo = new DocumentParseInfo();
+        documentParseInfo.documentId = documentId;
+        while (true) {
+            // we should now be at the start of a feed operation or at the end of the feed
+            JsonToken t = nextToken(parser);
+            if (t == null) {
+                throw new IllegalArgumentException("Could not read document, no document?");
+            }
+            switch (t) {
+                case START_OBJECT:
+                    indentLevel++;
+                    break;
+                case END_OBJECT:
+                    indentLevel--;
+                    break;
+                case START_ARRAY:
+                    indentLevel+=10000L;
+                    break;
+                case END_ARRAY:
+                    indentLevel-=10000L;
+                    break;
+            }
+            if (indentLevel == 1 && (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE)) {
+                try {
+                    if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) {
+                        documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue());
+                        continue;
+                    }
+                } catch (IOException e) {
+                    throw new RuntimeException("Got IO exception while parsing document", e);
+                }
+            }
+            if (indentLevel == 2L && t == JsonToken.START_OBJECT) {
+
+                try {
+                    if (!FIELDS.equals(parser.getCurrentName())) {
+                        continue;
+                    }
+                } catch (IOException e) {
+                    throw new RuntimeException("Got IO exception while parsing document", e);
+                }
+                documentParseInfo.fieldsBuffer.bufferObject(t, parser);
+                break;
+            }
+        }
+        return documentParseInfo;
+    }
+
+    private static JsonToken nextToken(JsonParser parser) {
+        try {
+            return parser.nextValue();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java b/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java
new file mode 100644
index 00000000000..aa79e2dad44
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java
@@ -0,0 +1,80 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json.readers;
+
+import com.fasterxml.jackson.core.JsonToken;
+import com.google.common.base.Preconditions;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.json.TokenBuffer;
+import com.yahoo.document.update.FieldUpdate;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectCompositeEnd;
+import static com.yahoo.document.json.readers.WeightedSetReader.fillWeightedSetUpdate;
+
+public class AddRemoveCreator {
+
+    // yes, this suppresswarnings ugliness is by intention, the code relies on
+    // the contracts in the builders
+    @SuppressWarnings({ "cast", "rawtypes", "unchecked" })
+    public static void createAdds(TokenBuffer buffer, Field field, FieldUpdate update) {
+        createAddsOrRemoves(buffer, field, update, false);
+    }
+
+    // yes, this suppresswarnings ugliness is by intention, the code relies on
+    // the contracts in the builders
+    @SuppressWarnings({ "cast", "rawtypes", "unchecked" })
+    public static void createRemoves(TokenBuffer buffer, Field field, FieldUpdate update) {
+        createAddsOrRemoves(buffer, field, update, true);
+    }
+
+    // yes, this suppresswarnings ugliness is by intention, the code relies on
+    // the contracts in the builders
+    @SuppressWarnings({ "cast", "rawtypes", "unchecked" })
+    private static void createAddsOrRemoves(TokenBuffer buffer, Field field, FieldUpdate update, boolean isRemove) {
+        FieldValue container = field.getDataType().createFieldValue();
+        FieldUpdate singleUpdate;
+        int initNesting = buffer.nesting();
+        JsonToken token;
+
+        Preconditions.checkState(buffer.currentToken().isStructStart(), "Expected start of composite, got %s", buffer.currentToken());
+        if (container instanceof CollectionFieldValue) {
+            token = buffer.next();
+            DataType valueType = ((CollectionFieldValue) container).getDataType().getNestedType();
+            if (container instanceof WeightedSet) {
+                // these are objects with string keys (which are the nested
+                // types) and values which are the weight
+                WeightedSet weightedSet = (WeightedSet) container;
+                fillWeightedSetUpdate(buffer, initNesting, valueType, weightedSet);
+                if (isRemove) {
+                    singleUpdate = FieldUpdate.createRemoveAll(field, weightedSet);
+                } else {
+                    singleUpdate = FieldUpdate.createAddAll(field, weightedSet);
+
+                }
+            } else {
+                List<FieldValue> arrayContents = new ArrayList<>();
+                token = ArrayReader.fillArrayUpdate(buffer, initNesting, token, valueType, arrayContents);
+                if (token != JsonToken.END_ARRAY) {
+                    throw new IllegalStateException("Expected END_ARRAY. Got '" + token + "'.");
+                }
+                if (isRemove) {
+                    singleUpdate = FieldUpdate.createRemoveAll(field, arrayContents);
+                } else {
+                    singleUpdate = FieldUpdate.createAddAll(field, arrayContents);
+                }
+            }
+        } else {
+            throw new UnsupportedOperationException(
+                    "Trying to add or remove from a field of a type the reader does not know how to handle: "
+                            + container.getClass().getName());
+        }
+        expectCompositeEnd(buffer.currentToken());
+        update.addAll(singleUpdate);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java
new file mode 100644
index 00000000000..ab3c3e23816
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java
@@ -0,0 +1,35 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json.readers;
+
+import com.fasterxml.jackson.core.JsonToken;
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.json.TokenBuffer;
+
+import java.util.List;
+
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart;
+import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue;
+
+public class ArrayReader {
+    static public JsonToken fillArrayUpdate(TokenBuffer buffer, int initNesting, JsonToken initToken, DataType valueType, List<FieldValue> arrayContents) {
+        JsonToken token = initToken;
+        while (buffer.nesting() >= initNesting) {
+            arrayContents.add(readSingleValue(buffer, token, valueType));
+            token = buffer.next();
+        }
+        return token;
+    }
+
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    public static void fillArray(TokenBuffer buffer, CollectionFieldValue parent, DataType valueType) {
+        int initNesting = buffer.nesting();
+        expectArrayStart(buffer.currentToken());
+        JsonToken token = buffer.next();
+        while (buffer.nesting() >= initNesting) {
+            parent.add(readSingleValue(buffer, token, valueType));
+            token = buffer.next();
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java
new file mode 100644
index 00000000000..b9f4d1d4613
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java
@@ -0,0 +1,60 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json.readers;
+
+import com.fasterxml.jackson.core.JsonToken;
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.json.TokenBuffer;
+
+import static com.yahoo.document.json.readers.ArrayReader.fillArray;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectCompositeEnd;
+import static com.yahoo.document.json.readers.WeightedSetReader.fillWeightedSet;
+
+public class CompositeReader {
+
+    // TODO createComposite is extremely similar to add/remove, refactor
+    // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders
+    @SuppressWarnings({ "cast", "rawtypes" })
+    public static FieldValue createComposite(TokenBuffer buffer, DataType expectedType) {
+        FieldValue fieldValue = expectedType.createFieldValue();
+        populateComposite(buffer, fieldValue);
+        return fieldValue;
+    }
+
+    // TODO createComposite is extremely similar to add/remove, refactor
+    // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders
+    @SuppressWarnings({ "cast", "rawtypes" })
+    public static void populateComposite(TokenBuffer buffer, FieldValue fieldValue) {
+        JsonToken token = buffer.currentToken();
+        if ((token != JsonToken.START_OBJECT) && (token != JsonToken.START_ARRAY)) {
+            throw new IllegalArgumentException("Expected '[' or '{'. Got '" + token + "'.");
+        }
+        if (fieldValue instanceof CollectionFieldValue) {
+            DataType valueType = ((CollectionFieldValue) fieldValue).getDataType().getNestedType();
+            if (fieldValue instanceof WeightedSet) {
+                fillWeightedSet(buffer, valueType, (WeightedSet) fieldValue);
+            } else {
+                fillArray(buffer, (CollectionFieldValue) fieldValue, valueType);
+            }
+        } else if (fieldValue instanceof MapFieldValue) {
+            MapReader.fillMap(buffer, (MapFieldValue) fieldValue);
+        } else if (fieldValue instanceof StructuredFieldValue) {
+            StructReader.fillStruct(buffer, (StructuredFieldValue) fieldValue);
+        } else if (fieldValue instanceof TensorFieldValue) {
+            TensorReader.fillTensor(buffer, (TensorFieldValue) fieldValue);
+        } else {
+            throw new IllegalStateException("Has created a composite field"
+                    + " value the reader does not know how to handle: "
+                    + fieldValue.getClass().getName() + " This is a bug. token = " + token);
+        }
+        expectCompositeEnd(buffer.currentToken());
+    }
+
+
+
+}
diff --git a/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java
new file mode 100644
index 00000000000..7401e33c607
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java
@@ -0,0 +1,16 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json.readers;
+
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.json.TokenBuffer;
+import com.yahoo.document.json.document.DocumentParser;
+
+import java.util.Optional;
+
+public class DocumentParseInfo {
+    public DocumentId documentId;
+    public Optional<Boolean> create = Optional.empty();
+    public Optional<String> condition = Optional.empty();
+    public DocumentParser.SupportedOperation operationType = null;
+    public TokenBuffer fieldsBuffer = new TokenBuffer();
+}
diff --git a/document/src/main/java/com/yahoo/document/json/readers/JsonParserHelpers.java b/document/src/main/java/com/yahoo/document/json/readers/JsonParserHelpers.java
new file mode 100644
index 00000000000..75e0860e2d5
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/readers/JsonParserHelpers.java
@@ -0,0 +1,23 @@
+package com.yahoo.document.json.readers;
+
+
+import com.fasterxml.jackson.core.JsonToken;
+import com.google.common.base.Preconditions;
+
+public class JsonParserHelpers {
+    public static void expectArrayStart(JsonToken token) {
+        Preconditions.checkState(token == JsonToken.START_ARRAY, "Expected start of array, got %s", token);
+    }
+
+    public static void expectObjectStart(JsonToken token) {
+        Preconditions.checkState(token == JsonToken.START_OBJECT, "Expected start of JSON object, got %s", token);
+    }
+
+    public static void expectObjectEnd(JsonToken token) {
+        Preconditions.checkState(token == JsonToken.END_OBJECT, "Expected end of JSON object, got %s", token);
+    }
+
+    public static void expectCompositeEnd(JsonToken token) {
+        Preconditions.checkState(token.isStructEnd(), "Expected end of composite, got %s", token);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/json/readers/MapReader.java b/document/src/main/java/com/yahoo/document/json/readers/MapReader.java
new file mode 100644
index 00000000000..0897f05e82a
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/readers/MapReader.java
@@ -0,0 +1,126 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json.readers;
+
+import com.fasterxml.jackson.core.JsonToken;
+import com.google.common.base.Preconditions;
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+import com.yahoo.document.json.TokenBuffer;
+import com.yahoo.document.update.MapValueUpdate;
+import com.yahoo.document.update.ValueUpdate;
+
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectEnd;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart;
+import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate;
+import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue;
+
+public class MapReader {
+    public static final String MAP_KEY = "key";
+    public static final String MAP_VALUE = "value";
+    public static final String UPDATE_ELEMENT = "element";
+    public static final String UPDATE_MATCH = "match";
+
+    @SuppressWarnings({ "rawtypes", "cast", "unchecked" })
+    public static void fillMap(TokenBuffer buffer, MapFieldValue parent) {
+        JsonToken token = buffer.currentToken();
+        int initNesting = buffer.nesting();
+        expectArrayStart(token);
+        token = buffer.next();
+        DataType keyType = parent.getDataType().getKeyType();
+        DataType valueType = parent.getDataType().getValueType();
+        while (buffer.nesting() >= initNesting) {
+            FieldValue key = null;
+            FieldValue value = null;
+            expectObjectStart(token);
+            token = buffer.next();
+            for (int i = 0; i < 2; ++i) {
+                if (MAP_KEY.equals(buffer.currentName())) {
+                    key = readSingleValue(buffer, token, keyType);
+                } else if (MAP_VALUE.equals(buffer.currentName())) {
+                    value = readSingleValue(buffer, token, valueType);
+                }
+                token = buffer.next();
+            }
+            Preconditions.checkState(key != null && value != null, "Missing key or value for map entry.");
+            parent.put(key, value);
+
+            expectObjectEnd(token);
+            token = buffer.next(); // array end or next entry
+        }
+    }
+
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    public static ValueUpdate createMapUpdate(TokenBuffer buffer, DataType currentLevel, FieldValue keyParent, FieldValue topLevelKey) {
+        TokenBuffer.Token element = buffer.prefetchScalar(UPDATE_ELEMENT);
+        if (UPDATE_ELEMENT.equals(buffer.currentName())) {
+            buffer.next();
+        }
+
+        FieldValue key = keyTypeForMapUpdate(element, currentLevel);
+        if (keyParent != null) {
+            ((CollectionFieldValue) keyParent).add(key);
+        }
+        // structure is: [(match + element)*, (element + action)]
+        // match will always have element, and either match or action
+        if (!UPDATE_MATCH.equals(buffer.currentName())) {
+            // we have reached an action...
+            if (topLevelKey == null) {
+                return ValueUpdate.createMap(key, readSingleUpdate(buffer, valueTypeForMapUpdate(currentLevel), buffer.currentName()));
+            } else {
+                return ValueUpdate.createMap(topLevelKey, readSingleUpdate(buffer, valueTypeForMapUpdate(currentLevel), buffer.currentName()));
+            }
+        } else {
+            // next level of matching
+            if (topLevelKey == null) {
+                return createMapUpdate(buffer, valueTypeForMapUpdate(currentLevel), key, key);
+            } else {
+                return createMapUpdate(buffer, valueTypeForMapUpdate(currentLevel), key, topLevelKey);
+            }
+        }
+    }
+
+    @SuppressWarnings("rawtypes")
+    public static ValueUpdate createMapUpdate(TokenBuffer buffer, Field field) {
+        buffer.next();
+        MapValueUpdate m = (MapValueUpdate) MapReader.createMapUpdate(buffer, field.getDataType(), null, null);
+        buffer.next();
+        // must generate the field value in parallell with the actual
+        return m;
+
+    }
+
+    private static DataType valueTypeForMapUpdate(DataType parentType) {
+        if (parentType instanceof WeightedSetDataType) {
+            return DataType.INT;
+        } else if (parentType instanceof CollectionDataType) {
+            return ((CollectionDataType) parentType).getNestedType();
+        } else if (parentType instanceof MapDataType) {
+            return ((MapDataType) parentType).getValueType();
+        } else {
+            throw new UnsupportedOperationException("Unexpected parent type: " + parentType);
+        }
+    }
+
+    private static FieldValue keyTypeForMapUpdate(TokenBuffer.Token element, DataType expectedType) {
+        FieldValue v;
+        if (expectedType instanceof ArrayDataType) {
+            v = new IntegerFieldValue(Integer.valueOf(element.text));
+        } else if (expectedType instanceof WeightedSetDataType) {
+            v = ((WeightedSetDataType) expectedType).getNestedType().createFieldValue(element.text);
+        } else if (expectedType instanceof MapDataType) {
+            v = ((MapDataType) expectedType).getKeyType().createFieldValue(element.text);
+        } else {
+            throw new IllegalArgumentException("Container type " + expectedType + " not supported for match update.");
+        }
+        return v;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java
new file mode 100644
index 00000000000..24ad5f0589a
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java
@@ -0,0 +1,78 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json.readers;
+
+import com.fasterxml.jackson.core.JsonToken;
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.ReferenceDataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.json.TokenBuffer;
+import com.yahoo.document.update.ValueUpdate;
+import org.apache.commons.codec.binary.Base64;
+
+public class SingleValueReader {
+    public static final String UPDATE_ASSIGN = "assign";
+    public static final String UPDATE_INCREMENT = "increment";
+    public static final String UPDATE_DECREMENT = "decrement";
+    public static final String UPDATE_MULTIPLY = "multiply";
+    public static final String UPDATE_DIVIDE = "divide";
+
+    public static FieldValue readSingleValue(TokenBuffer buffer, JsonToken t, DataType expectedType) {
+        if (t.isScalarValue()) {
+            return readAtomic(buffer, expectedType);
+        } else {
+            return CompositeReader.createComposite(buffer, expectedType);
+        }
+    }
+
+    @SuppressWarnings("rawtypes")
+    public static ValueUpdate readSingleUpdate(TokenBuffer buffer, DataType expectedType, String action) {
+        ValueUpdate update;
+
+        switch (action) {
+            case UPDATE_ASSIGN:
+                update = (buffer.currentToken() == JsonToken.VALUE_NULL)
+                        ? ValueUpdate.createClear()
+                        : ValueUpdate.createAssign(readSingleValue(buffer, buffer.currentToken(), expectedType));
+                break;
+            // double is silly, but it's what is used internally anyway
+            case UPDATE_INCREMENT:
+                update = ValueUpdate.createIncrement(Double.valueOf(buffer.currentText()));
+                break;
+            case UPDATE_DECREMENT:
+                update = ValueUpdate.createDecrement(Double.valueOf(buffer.currentText()));
+                break;
+            case UPDATE_MULTIPLY:
+                update = ValueUpdate.createMultiply(Double.valueOf(buffer.currentText()));
+                break;
+            case UPDATE_DIVIDE:
+                update = ValueUpdate.createDivide(Double.valueOf(buffer.currentText()));
+                break;
+            default:
+                throw new IllegalArgumentException("Operation \"" + buffer.currentName() + "\" not implemented.");
+        }
+        return update;
+    }
+
+    public static FieldValue readAtomic(TokenBuffer buffer, DataType expectedType) {
+        if (expectedType.equals(DataType.RAW)) {
+            return expectedType.createFieldValue(new Base64().decode(buffer.currentText()));
+        } else if (expectedType.equals(PositionDataType.INSTANCE)) {
+            return PositionDataType.fromString(buffer.currentText());
+        } else if (expectedType instanceof ReferenceDataType) {
+            return readReferenceFieldValue(buffer, expectedType);
+        } else {
+            return expectedType.createFieldValue(buffer.currentText());
+        }
+    }
+
+    private static FieldValue readReferenceFieldValue(TokenBuffer buffer, DataType expectedType) {
+        final FieldValue value = expectedType.createFieldValue();
+        final String refText = buffer.currentText();
+        if (!refText.isEmpty()) {
+            value.assign(new DocumentId(buffer.currentText()));
+        }
+        return value;
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java
new file mode 100644
index 00000000000..17c9123e1cd
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java
@@ -0,0 +1,40 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json.readers;
+
+import com.fasterxml.jackson.core.JsonToken;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+import com.yahoo.document.json.JsonReaderException;
+import com.yahoo.document.json.TokenBuffer;
+
+import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue;
+
+public class StructReader {
+    public static void fillStruct(TokenBuffer buffer, StructuredFieldValue parent) {
+        // do note the order of initializing initNesting and token is relevant for empty docs
+        int initNesting = buffer.nesting();
+        JsonToken token = buffer.next();
+
+        while (buffer.nesting() >= initNesting) {
+            Field f = getField(buffer, parent);
+            try {
+                FieldValue v = readSingleValue(buffer, token, f.getDataType());
+                parent.setFieldValue(f, v);
+                token = buffer.next();
+            } catch (IllegalArgumentException e) {
+                throw new JsonReaderException(f, e);
+            }
+        }
+    }
+
+    public static Field getField(TokenBuffer buffer, StructuredFieldValue parent) {
+        Field f = parent.getField(buffer.currentName());
+        if (f == null) {
+            throw new NullPointerException("Could not get field \"" + buffer.currentName() +
+                    "\" in the structure of type \"" + parent.getDataType().getDataTypeName() + "\".");
+        }
+        return f;
+    }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java
new file mode 100644
index 00000000000..5dd32753ed6
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java
@@ -0,0 +1,65 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json.readers;
+
+import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.document.json.TokenBuffer;
+import com.yahoo.tensor.MappedTensor;
+import com.yahoo.tensor.Tensor;
+
+import static com.yahoo.document.json.readers.JsonParserHelpers.*;
+
+public class TensorReader {
+    public static final String TENSOR_ADDRESS = "address";
+    public static final String TENSOR_DIMENSIONS = "dimensions";
+    public static final String TENSOR_CELLS = "cells";
+    public static final String TENSOR_VALUE = "value";
+
+    public static void fillTensor(TokenBuffer buffer, TensorFieldValue tensorFieldValue) {
+        Tensor.Builder tensorBuilder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType());
+        expectObjectStart(buffer.currentToken());
+        int initNesting = buffer.nesting();
+        /* read tensor cell fields and ignore everything else */
+        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
+            if (TensorReader.TENSOR_CELLS.equals(buffer.currentName()))
+                readTensorCells(buffer, tensorBuilder);
+        }
+        expectObjectEnd(buffer.currentToken());
+        tensorFieldValue.assign(tensorBuilder.build());
+    }
+
+    public static void readTensorCells(TokenBuffer buffer, Tensor.Builder tensorBuilder) {
+        expectArrayStart(buffer.currentToken());
+        int initNesting = buffer.nesting();
+        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next())
+            readTensorCell(buffer, tensorBuilder);
+        expectCompositeEnd(buffer.currentToken());
+    }
+
+    public static void readTensorCell(TokenBuffer buffer, Tensor.Builder tensorBuilder) {
+        expectObjectStart(buffer.currentToken());
+        int initNesting = buffer.nesting();
+        double cellValue = 0.0;
+        Tensor.Builder.CellBuilder cellBuilder = tensorBuilder.cell();
+        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
+            String currentName = buffer.currentName();
+            if (TensorReader.TENSOR_ADDRESS.equals(currentName)) {
+                readTensorAddress(buffer, cellBuilder);
+            } else if (TensorReader.TENSOR_VALUE.equals(currentName)) {
+                cellValue = Double.valueOf(buffer.currentText());
+            }
+        }
+        expectObjectEnd(buffer.currentToken());
+        cellBuilder.value(cellValue);
+    }
+
+    public static void readTensorAddress(TokenBuffer buffer, MappedTensor.Builder.CellBuilder cellBuilder) {
+        expectObjectStart(buffer.currentToken());
+        int initNesting = buffer.nesting();
+        for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
+            String dimension = buffer.currentName();
+            String label = buffer.currentText();
+            cellBuilder.label(dimension, label);
+        }
+        expectObjectEnd(buffer.currentToken());
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java b/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java
new file mode 100644
index 00000000000..af4f5a8d226
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java
@@ -0,0 +1,32 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json.readers;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.json.TokenBuffer;
+
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart;
+
+
+public class WeightedSetReader {
+    public static void fillWeightedSet(TokenBuffer buffer, DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) {
+        int initNesting = buffer.nesting();
+        expectObjectStart(buffer.currentToken());
+        buffer.next();
+        iterateThroughWeightedSet(buffer, initNesting, valueType, weightedSet);
+    }
+    public static void fillWeightedSetUpdate(TokenBuffer buffer, int initNesting, DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) {
+        iterateThroughWeightedSet(buffer, initNesting, valueType, weightedSet);
+    }
+
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    private static void iterateThroughWeightedSet(TokenBuffer buffer, int initNesting, DataType valueType, WeightedSet weightedSet) {
+        while (buffer.nesting() >= initNesting) {
+            // XXX the keys are defined in the spec to always be represented as strings
+            FieldValue v = valueType.createFieldValue(buffer.currentName());
+            weightedSet.put(v, Integer.valueOf(buffer.currentText()));
+            buffer.next();
+        }
+    }
+}