Move more things around.

13 files changed, 172 insertions, 166 deletions

diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java
index 4fc20649cc6..2d085f30eac 100644
--- a/document/src/main/java/com/yahoo/document/json/JsonReader.java
+++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java
@@ -6,7 +6,6 @@ import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonToken;
 import com.google.common.annotations.Beta;
 import com.google.common.base.Preconditions;
-import com.yahoo.document.DataType;
 import com.yahoo.document.Document;
 import com.yahoo.document.DocumentId;
 import com.yahoo.document.DocumentOperation;
@@ -16,26 +15,23 @@ import com.yahoo.document.DocumentType;
 import com.yahoo.document.DocumentTypeManager;
 import com.yahoo.document.DocumentUpdate;
 import com.yahoo.document.Field;
-import com.yahoo.document.PositionDataType;
-import com.yahoo.document.ReferenceDataType;
 import com.yahoo.document.TestAndSetCondition;
-import com.yahoo.document.datatypes.FieldValue;
-import com.yahoo.document.datatypes.StructuredFieldValue;
 import com.yahoo.document.json.document.DocumentParser;
 import com.yahoo.document.json.readers.DocumentParseInfo;
-import com.yahoo.document.json.readers.DocumentToFields;
-import com.yahoo.document.json.readers.MapReader;
 import com.yahoo.document.update.FieldUpdate;
-import com.yahoo.document.update.MapValueUpdate;
-import com.yahoo.document.update.ValueUpdate;
-import org.apache.commons.codec.binary.Base64;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Optional;
 
+import static com.yahoo.document.json.document.DocumentParser.parseToDocumentsFieldsAndInsertFieldsIntoBuffer;
 import static com.yahoo.document.json.readers.AddRemoveCreator.createAddsOrRemoves;
 import static com.yahoo.document.json.readers.CompositeReader.populateComposite;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectEnd;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart;
+import static com.yahoo.document.json.readers.MapReader.UPDATE_MATCH;
+import static com.yahoo.document.json.readers.MapReader.createMapUpdate;
 import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate;
 
 /**
@@ -48,6 +44,7 @@ import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate
 @Beta
 public class JsonReader {
 
+    // Only used for testing.
     public Optional<DocumentParseInfo> parseDocument() {
         return DocumentParser.parseDocument(parser);
     }
@@ -55,15 +52,10 @@ public class JsonReader {
     public enum FieldOperation {
         ADD, REMOVE
     }
-    public static final String FIELDS = "fields";
-    public static final String REMOVE = "remove";
 
-    public static final String CONDITION = "condition";
-    public static final String CREATE_IF_NON_EXISTENT = "create";
+
     private static final String UPDATE_REMOVE = "remove";
-    public static final String UPDATE_MATCH = "match";
     private static final String UPDATE_ADD = "add";
-    public static final String UPDATE_ELEMENT = "element";
 
     private final JsonParser parser;
     private final DocumentTypeManager typeManager;
@@ -96,7 +88,7 @@ public class JsonReader {
      */
     public DocumentOperation readSingleDocument(SupportedOperation operationType, String docIdString) {
         DocumentId docId = new DocumentId(docIdString);
-        DocumentParseInfo documentParseInfo = DocumentToFields.parseToDocumentsFieldsAndInsertFieldsIntoBuffer(parser, docId);
+        DocumentParseInfo documentParseInfo = parseToDocumentsFieldsAndInsertFieldsIntoBuffer(parser, docId);
         documentParseInfo.operationType = operationType;
         DocumentOperation operation = createDocumentOperation(documentParseInfo.fieldsBuffer, documentParseInfo);
         operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.condition));
@@ -161,6 +153,7 @@ public class JsonReader {
         return documentOperation;
     }
 
+    // Exposed for unit testing...
     void readUpdate(TokenBuffer buffer, DocumentUpdate next) {
         if (buffer.size() == 0) {
             bufferFields(parser, buffer, nextToken(parser));
@@ -168,6 +161,7 @@ public class JsonReader {
         populateUpdateFromBuffer(buffer, next);
     }
 
+    // Exposed for unit testing...
     void readPut(TokenBuffer buffer, DocumentPut put) {
         if (buffer.size() == 0) {
             bufferFields(parser, buffer, nextToken(parser));
@@ -186,7 +180,7 @@ public class JsonReader {
         Preconditions.checkState(buffer.size() == 0, "Dangling data at end of operation");
     }
 
-    private void populateUpdateFromBuffer(TokenBuffer buffer, DocumentUpdate update) {
+    private static void populateUpdateFromBuffer(TokenBuffer buffer, DocumentUpdate update) {
         expectObjectStart(buffer.currentToken());
         int localNesting = buffer.nesting();
         JsonToken t = buffer.next();
@@ -200,7 +194,7 @@ public class JsonReader {
         }
     }
 
-    private void addFieldUpdates(TokenBuffer buffer, DocumentUpdate update, Field field) {
+    private static void addFieldUpdates(TokenBuffer buffer, DocumentUpdate update, Field field) {
         int localNesting = buffer.nesting();
         FieldUpdate fieldUpdate = FieldUpdate.create(field);
 
@@ -225,62 +219,6 @@ public class JsonReader {
         update.addFieldUpdate(fieldUpdate);
     }
 
-    @SuppressWarnings("rawtypes")
-    public ValueUpdate createMapUpdate(TokenBuffer buffer, Field field) {
-        buffer.next();
-        MapValueUpdate m = (MapValueUpdate) MapReader.createMapUpdate(buffer, field.getDataType(), null, null);
-        buffer.next();
-        // must generate the field value in parallell with the actual
-        return m;
-
-    }
-
-    public static void expectCompositeEnd(JsonToken token) {
-        Preconditions.checkState(token.isStructEnd(), "Expected end of composite, got %s", token);
-    }
-
-    public static Field getField(TokenBuffer buffer, StructuredFieldValue parent) {
-        Field f = parent.getField(buffer.currentName());
-        if (f == null) {
-            throw new NullPointerException("Could not get field \"" + buffer.currentName() +
-                    "\" in the structure of type \"" + parent.getDataType().getDataTypeName() + "\".");
-        }
-        return f;
-    }
-
-    public static void expectArrayStart(JsonToken token) {
-        Preconditions.checkState(token == JsonToken.START_ARRAY, "Expected start of array, got %s", token);
-    }
-
-    public static void expectObjectStart(JsonToken token) {
-        Preconditions.checkState(token == JsonToken.START_OBJECT, "Expected start of JSON object, got %s", token);
-    }
-
-    public static void expectObjectEnd(JsonToken token) {
-        Preconditions.checkState(token == JsonToken.END_OBJECT, "Expected end of JSON object, got %s", token);
-    }
-
-    public static FieldValue readAtomic(TokenBuffer buffer, DataType expectedType) {
-        if (expectedType.equals(DataType.RAW)) {
-            return expectedType.createFieldValue(new Base64().decode(buffer.currentText()));
-        } else if (expectedType.equals(PositionDataType.INSTANCE)) {
-            return PositionDataType.fromString(buffer.currentText());
-        } else if (expectedType instanceof ReferenceDataType) {
-            return readReferenceFieldValue(buffer, expectedType);
-        } else {
-            return expectedType.createFieldValue(buffer.currentText());
-        }
-    }
-
-    private static FieldValue readReferenceFieldValue(TokenBuffer buffer, DataType expectedType) {
-        final FieldValue value = expectedType.createFieldValue();
-        final String refText = buffer.currentText();
-        if (!refText.isEmpty()) {
-            value.assign(new DocumentId(buffer.currentText()));
-        }
-        return value;
-    }
-
     public static void bufferFields(JsonParser parser, TokenBuffer buffer, JsonToken current) {
         buffer.bufferObject(current, parser);
     }
diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java
index 6ad526a2aaa..bc90141747f 100644
--- a/document/src/main/java/com/yahoo/document/json/JsonWriter.java
+++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java
@@ -38,6 +38,8 @@ import java.util.Iterator;
 import java.util.Map;
 
 import static com.yahoo.document.json.JsonSerializationHelper.*;
+import static com.yahoo.document.json.document.DocumentParser.FIELDS;
+import static com.yahoo.document.json.document.DocumentParser.REMOVE;
 
 /**
  * Serialize Document and other FieldValue instances as JSON.
@@ -106,7 +108,7 @@ public class JsonWriter implements DocumentWriter {
             // this makes it impossible to refeed directly, not sure what's correct
             // perhaps just change to "put"?
             generator.writeStringField("id", value.getId().toString());
-            generator.writeObjectFieldStart(JsonReader.FIELDS);
+            generator.writeObjectFieldStart(FIELDS);
 
             Iterator<Map.Entry<Field, FieldValue>> i = value.iterator();
             while (i.hasNext()) {
@@ -251,7 +253,7 @@ public class JsonWriter implements DocumentWriter {
         try {
             JsonGenerator throwAway = jsonFactory.createGenerator(out);
             throwAway.writeStartObject();
-            throwAway.writeStringField(JsonReader.REMOVE, docId.toString());
+            throwAway.writeStringField(REMOVE, docId.toString());
             throwAway.writeEndObject();
             throwAway.close();
         } catch (IOException e) {
diff --git a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java
index deac5498f5a..8ac56e29b94 100644
--- a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java
+++ b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java
@@ -4,17 +4,24 @@ package com.yahoo.document.json.document;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonToken;
 import com.yahoo.document.DocumentId;
+import com.yahoo.document.json.JsonReader;
 import com.yahoo.document.json.readers.DocumentParseInfo;
 
 import java.io.IOException;
 import java.util.Optional;
 
-import static com.yahoo.document.json.JsonReader.*;
+import static com.yahoo.document.json.JsonReader.bufferFields;
+import static com.yahoo.document.json.JsonReader.nextToken;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart;
 
 public class DocumentParser {
     private static final String UPDATE = "update";
     private static final String PUT = "put";
     private static final String ID = "id";
+    private static final String CONDITION = "condition";
+    public static final String CREATE_IF_NON_EXISTENT = "create";
+    public static final String FIELDS = "fields";
+    public static final String REMOVE = "remove";
 
     public static Optional<DocumentParseInfo> parseDocument(JsonParser parser) {
         // we should now be at the start of a feed operation or at the end of the feed
@@ -68,19 +75,70 @@ public class DocumentParser {
         }
     }
 
-    private static SupportedOperation operationNameToOperationType(String operationName) {
+    private static JsonReader.SupportedOperation operationNameToOperationType(String operationName) {
         switch (operationName) {
             case PUT:
             case ID:
-                return SupportedOperation.PUT;
+                return JsonReader.SupportedOperation.PUT;
             case REMOVE:
-                return SupportedOperation.REMOVE;
+                return JsonReader.SupportedOperation.REMOVE;
             case UPDATE:
-                return SupportedOperation.UPDATE;
+                return JsonReader.SupportedOperation.UPDATE;
             default:
                 throw new IllegalArgumentException(
                         "Got " + operationName + " as document operation, only \"put\", " +
                                 "\"remove\" and \"update\" are supported.");
         }
     }
+
+
+    public static DocumentParseInfo parseToDocumentsFieldsAndInsertFieldsIntoBuffer(JsonParser parser, DocumentId documentId) {
+        long indentLevel = 0;
+        DocumentParseInfo documentParseInfo = new DocumentParseInfo();
+        documentParseInfo.documentId = documentId;
+        while (true) {
+            // we should now be at the start of a feed operation or at the end of the feed
+            JsonToken t = nextToken(parser);
+            if (t == null) {
+                throw new IllegalArgumentException("Could not read document, no document?");
+            }
+            switch (t) {
+                case START_OBJECT:
+                    indentLevel++;
+                    break;
+                case END_OBJECT:
+                    indentLevel--;
+                    break;
+                case START_ARRAY:
+                    indentLevel+=10000L;
+                    break;
+                case END_ARRAY:
+                    indentLevel-=10000L;
+                    break;
+            }
+            if (indentLevel == 1 && (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE)) {
+                try {
+                    if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) {
+                        documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue());
+                        continue;
+                    }
+                } catch (IOException e) {
+                    throw new RuntimeException("Got IO exception while parsing document", e);
+                }
+            }
+            if (indentLevel == 2L && t == JsonToken.START_OBJECT) {
+
+                try {
+                    if (!FIELDS.equals(parser.getCurrentName())) {
+                        continue;
+                    }
+                } catch (IOException e) {
+                    throw new RuntimeException("Got IO exception while parsing document", e);
+                }
+                bufferFields(parser, documentParseInfo.fieldsBuffer, t);
+                break;
+            }
+        }
+        return documentParseInfo;
+    }
 }
diff --git a/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java b/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java
index e353da05ad1..022c35a7d85 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java
@@ -15,7 +15,7 @@ import com.yahoo.document.update.FieldUpdate;
 import java.util.ArrayList;
 import java.util.List;
 
-import static com.yahoo.document.json.JsonReader.expectCompositeEnd;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectCompositeEnd;
 import static com.yahoo.document.json.readers.WeightedSetReader.fillWeightedSetUpdate;
 
 public class AddRemoveCreator {
diff --git a/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java
index 6f216bbca53..ab3c3e23816 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java
@@ -9,7 +9,7 @@ import com.yahoo.document.json.TokenBuffer;
 
 import java.util.List;
 
-import static com.yahoo.document.json.JsonReader.expectArrayStart;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart;
 import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue;
 
 public class ArrayReader {
diff --git a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java
index ba5fcab2f01..b9f4d1d4613 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java
@@ -11,8 +11,8 @@ import com.yahoo.document.datatypes.TensorFieldValue;
 import com.yahoo.document.datatypes.WeightedSet;
 import com.yahoo.document.json.TokenBuffer;
 
-import static com.yahoo.document.json.JsonReader.expectCompositeEnd;
 import static com.yahoo.document.json.readers.ArrayReader.fillArray;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectCompositeEnd;
 import static com.yahoo.document.json.readers.WeightedSetReader.fillWeightedSet;
 
 public class CompositeReader {
diff --git a/document/src/main/java/com/yahoo/document/json/readers/DocumentToFields.java b/document/src/main/java/com/yahoo/document/json/readers/DocumentToFields.java
deleted file mode 100644
index fcebe510b5b..00000000000
--- a/document/src/main/java/com/yahoo/document/json/readers/DocumentToFields.java
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.document.json.readers;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonToken;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.json.JsonReader;
-
-import java.io.IOException;
-import java.util.Optional;
-
-import static com.yahoo.document.json.JsonReader.CREATE_IF_NON_EXISTENT;
-import static com.yahoo.document.json.JsonReader.FIELDS;
-
-public class DocumentToFields {
-    public static DocumentParseInfo parseToDocumentsFieldsAndInsertFieldsIntoBuffer(JsonParser parser, DocumentId documentId) {
-        long indentLevel = 0;
-        DocumentParseInfo documentParseInfo = new DocumentParseInfo();
-        documentParseInfo.documentId = documentId;
-        while (true) {
-            // we should now be at the start of a feed operation or at the end of the feed
-            JsonToken t = JsonReader.nextToken(parser);
-            if (t == null) {
-                throw new IllegalArgumentException("Could not read document, no document?");
-            }
-            switch (t) {
-                case START_OBJECT:
-                    indentLevel++;
-                    break;
-                case END_OBJECT:
-                    indentLevel--;
-                    break;
-                case START_ARRAY:
-                    indentLevel+=10000L;
-                    break;
-                case END_ARRAY:
-                    indentLevel-=10000L;
-                    break;
-            }
-            if (indentLevel == 1 && (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE)) {
-                try {
-                    if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) {
-                        documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue());
-                        continue;
-                    }
-                } catch (IOException e) {
-                    throw new RuntimeException("Got IO exception while parsing document", e);
-                }
-            }
-            if (indentLevel == 2L && t == JsonToken.START_OBJECT) {
-
-                try {
-                    if (!FIELDS.equals(parser.getCurrentName())) {
-                        continue;
-                    }
-                } catch (IOException e) {
-                    throw new RuntimeException("Got IO exception while parsing document", e);
-                }
-                JsonReader.bufferFields(parser, documentParseInfo.fieldsBuffer, t);
-                break;
-            }
-        }
-        return documentParseInfo;
-    }
-}
diff --git a/document/src/main/java/com/yahoo/document/json/readers/JsonParserHelpers.java b/document/src/main/java/com/yahoo/document/json/readers/JsonParserHelpers.java
new file mode 100644
index 00000000000..75e0860e2d5
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/readers/JsonParserHelpers.java
@@ -0,0 +1,23 @@
+package com.yahoo.document.json.readers;
+
+
+import com.fasterxml.jackson.core.JsonToken;
+import com.google.common.base.Preconditions;
+
+public class JsonParserHelpers {
+    public static void expectArrayStart(JsonToken token) {
+        Preconditions.checkState(token == JsonToken.START_ARRAY, "Expected start of array, got %s", token);
+    }
+
+    public static void expectObjectStart(JsonToken token) {
+        Preconditions.checkState(token == JsonToken.START_OBJECT, "Expected start of JSON object, got %s", token);
+    }
+
+    public static void expectObjectEnd(JsonToken token) {
+        Preconditions.checkState(token == JsonToken.END_OBJECT, "Expected end of JSON object, got %s", token);
+    }
+
+    public static void expectCompositeEnd(JsonToken token) {
+        Preconditions.checkState(token.isStructEnd(), "Expected end of composite, got %s", token);
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/json/readers/MapReader.java b/document/src/main/java/com/yahoo/document/json/readers/MapReader.java
index ad510924b9a..0897f05e82a 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/MapReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/MapReader.java
@@ -6,6 +6,7 @@ import com.google.common.base.Preconditions;
 import com.yahoo.document.ArrayDataType;
 import com.yahoo.document.CollectionDataType;
 import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
 import com.yahoo.document.MapDataType;
 import com.yahoo.document.WeightedSetDataType;
 import com.yahoo.document.datatypes.CollectionFieldValue;
@@ -13,15 +14,20 @@ import com.yahoo.document.datatypes.FieldValue;
 import com.yahoo.document.datatypes.IntegerFieldValue;
 import com.yahoo.document.datatypes.MapFieldValue;
 import com.yahoo.document.json.TokenBuffer;
+import com.yahoo.document.update.MapValueUpdate;
 import com.yahoo.document.update.ValueUpdate;
 
-import static com.yahoo.document.json.JsonReader.*;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectEnd;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart;
 import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate;
 import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue;
 
 public class MapReader {
     public static final String MAP_KEY = "key";
     public static final String MAP_VALUE = "value";
+    public static final String UPDATE_ELEMENT = "element";
+    public static final String UPDATE_MATCH = "match";
 
     @SuppressWarnings({ "rawtypes", "cast", "unchecked" })
     public static void fillMap(TokenBuffer buffer, MapFieldValue parent) {
@@ -82,6 +88,16 @@ public class MapReader {
         }
     }
 
+    @SuppressWarnings("rawtypes")
+    public static ValueUpdate createMapUpdate(TokenBuffer buffer, Field field) {
+        buffer.next();
+        MapValueUpdate m = (MapValueUpdate) MapReader.createMapUpdate(buffer, field.getDataType(), null, null);
+        buffer.next();
+        // must generate the field value in parallell with the actual
+        return m;
+
+    }
+
     private static DataType valueTypeForMapUpdate(DataType parentType) {
         if (parentType instanceof WeightedSetDataType) {
             return DataType.INT;
diff --git a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java
index 5440300eadf..24ad5f0589a 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java
@@ -3,10 +3,13 @@ package com.yahoo.document.json.readers;
 
 import com.fasterxml.jackson.core.JsonToken;
 import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.ReferenceDataType;
 import com.yahoo.document.datatypes.FieldValue;
-import com.yahoo.document.json.JsonReader;
 import com.yahoo.document.json.TokenBuffer;
 import com.yahoo.document.update.ValueUpdate;
+import org.apache.commons.codec.binary.Base64;
 
 public class SingleValueReader {
     public static final String UPDATE_ASSIGN = "assign";
@@ -17,7 +20,7 @@ public class SingleValueReader {
 
     public static FieldValue readSingleValue(TokenBuffer buffer, JsonToken t, DataType expectedType) {
         if (t.isScalarValue()) {
-            return JsonReader.readAtomic(buffer, expectedType);
+            return readAtomic(buffer, expectedType);
         } else {
             return CompositeReader.createComposite(buffer, expectedType);
         }
@@ -52,4 +55,24 @@ public class SingleValueReader {
         return update;
     }
 
+    public static FieldValue readAtomic(TokenBuffer buffer, DataType expectedType) {
+        if (expectedType.equals(DataType.RAW)) {
+            return expectedType.createFieldValue(new Base64().decode(buffer.currentText()));
+        } else if (expectedType.equals(PositionDataType.INSTANCE)) {
+            return PositionDataType.fromString(buffer.currentText());
+        } else if (expectedType instanceof ReferenceDataType) {
+            return readReferenceFieldValue(buffer, expectedType);
+        } else {
+            return expectedType.createFieldValue(buffer.currentText());
+        }
+    }
+
+    private static FieldValue readReferenceFieldValue(TokenBuffer buffer, DataType expectedType) {
+        final FieldValue value = expectedType.createFieldValue();
+        final String refText = buffer.currentText();
+        if (!refText.isEmpty()) {
+            value.assign(new DocumentId(buffer.currentText()));
+        }
+        return value;
+    }
 }
diff --git a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java
index 30620fb19bb..17c9123e1cd 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java
@@ -5,7 +5,6 @@ import com.fasterxml.jackson.core.JsonToken;
 import com.yahoo.document.Field;
 import com.yahoo.document.datatypes.FieldValue;
 import com.yahoo.document.datatypes.StructuredFieldValue;
-import com.yahoo.document.json.JsonReader;
 import com.yahoo.document.json.JsonReaderException;
 import com.yahoo.document.json.TokenBuffer;
 
@@ -18,7 +17,7 @@ public class StructReader {
         JsonToken token = buffer.next();
 
         while (buffer.nesting() >= initNesting) {
-            Field f = JsonReader.getField(buffer, parent);
+            Field f = getField(buffer, parent);
             try {
                 FieldValue v = readSingleValue(buffer, token, f.getDataType());
                 parent.setFieldValue(f, v);
@@ -28,4 +27,14 @@ public class StructReader {
             }
         }
     }
+
+    public static Field getField(TokenBuffer buffer, StructuredFieldValue parent) {
+        Field f = parent.getField(buffer.currentName());
+        if (f == null) {
+            throw new NullPointerException("Could not get field \"" + buffer.currentName() +
+                    "\" in the structure of type \"" + parent.getDataType().getDataTypeName() + "\".");
+        }
+        return f;
+    }
+
 }
diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java
index e4fa676a150..5dd32753ed6 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java
@@ -2,11 +2,12 @@
 package com.yahoo.document.json.readers;
 
 import com.yahoo.document.datatypes.TensorFieldValue;
-import com.yahoo.document.json.JsonReader;
 import com.yahoo.document.json.TokenBuffer;
 import com.yahoo.tensor.MappedTensor;
 import com.yahoo.tensor.Tensor;
 
+import static com.yahoo.document.json.readers.JsonParserHelpers.*;
+
 public class TensorReader {
     public static final String TENSOR_ADDRESS = "address";
     public static final String TENSOR_DIMENSIONS = "dimensions";
@@ -15,27 +16,27 @@ public class TensorReader {
 
     public static void fillTensor(TokenBuffer buffer, TensorFieldValue tensorFieldValue) {
         Tensor.Builder tensorBuilder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType());
-        JsonReader.expectObjectStart(buffer.currentToken());
+        expectObjectStart(buffer.currentToken());
         int initNesting = buffer.nesting();
-        // read tensor cell fields and ignore everything else
+        /* read tensor cell fields and ignore everything else */
         for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
             if (TensorReader.TENSOR_CELLS.equals(buffer.currentName()))
                 readTensorCells(buffer, tensorBuilder);
         }
-        JsonReader.expectObjectEnd(buffer.currentToken());
+        expectObjectEnd(buffer.currentToken());
         tensorFieldValue.assign(tensorBuilder.build());
     }
 
     public static void readTensorCells(TokenBuffer buffer, Tensor.Builder tensorBuilder) {
-        JsonReader.expectArrayStart(buffer.currentToken());
+        expectArrayStart(buffer.currentToken());
         int initNesting = buffer.nesting();
         for (buffer.next(); buffer.nesting() >= initNesting; buffer.next())
             readTensorCell(buffer, tensorBuilder);
-        JsonReader.expectCompositeEnd(buffer.currentToken());
+        expectCompositeEnd(buffer.currentToken());
     }
 
     public static void readTensorCell(TokenBuffer buffer, Tensor.Builder tensorBuilder) {
-        JsonReader.expectObjectStart(buffer.currentToken());
+        expectObjectStart(buffer.currentToken());
         int initNesting = buffer.nesting();
         double cellValue = 0.0;
         Tensor.Builder.CellBuilder cellBuilder = tensorBuilder.cell();
@@ -47,18 +48,18 @@ public class TensorReader {
                 cellValue = Double.valueOf(buffer.currentText());
             }
         }
-        JsonReader.expectObjectEnd(buffer.currentToken());
+        expectObjectEnd(buffer.currentToken());
         cellBuilder.value(cellValue);
     }
 
     public static void readTensorAddress(TokenBuffer buffer, MappedTensor.Builder.CellBuilder cellBuilder) {
-        JsonReader.expectObjectStart(buffer.currentToken());
+        expectObjectStart(buffer.currentToken());
         int initNesting = buffer.nesting();
         for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) {
             String dimension = buffer.currentName();
             String label = buffer.currentText();
             cellBuilder.label(dimension, label);
         }
-        JsonReader.expectObjectEnd(buffer.currentToken());
+        expectObjectEnd(buffer.currentToken());
     }
 }
diff --git a/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java b/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java
index ad0aaa10bf5..af4f5a8d226 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java
@@ -6,7 +6,8 @@ import com.yahoo.document.datatypes.FieldValue;
 import com.yahoo.document.datatypes.WeightedSet;
 import com.yahoo.document.json.TokenBuffer;
 
-import static com.yahoo.document.json.JsonReader.expectObjectStart;
+import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart;
+
 
 public class WeightedSetReader {
     public static void fillWeightedSet(TokenBuffer buffer, DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) {