Voffeloff/documentupdate json serializer (#675)

* Add JsonTestHelper

inputJson is a convenience method for inputing JSON in Java
assertJsonEquals tests if two JSON encoded strings are structurally
equal

* Hide overridden low-level interface so that users don't have to implement two interfaces

* Refactor methods common to JsonWriter and DocumentUpdateJsonSerializer into JsonHelper

* Remove wildcard imports

* DocumentUpdate JSON serializer

* Move static tensor serializing methods close to where their used and change access modifier to private

* Class renaming

JsonHelper -> JsonSerializationHelper
JsonSerializingException -> JsonSerializationException

* Style changes

* Remove default-failing implementations of Serializer in FieldWriter

Refactor common Serializer methods (and more) into
JsonSerializationHelper

* Identation/spacing changes

* Add final to some local variables

* JavaDoc

* Implement ClearValueUpdate

* Code review changes

5 files changed, 780 insertions, 276 deletions

diff --git a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java
new file mode 100644
index 00000000000..a059df01ad5
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java
@@ -0,0 +1,341 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.annotation.AnnotationReference;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.ByteFieldValue;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.serialization.DocumentUpdateWriter;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.update.AddValueUpdate;
+import com.yahoo.document.update.ArithmeticValueUpdate;
+import com.yahoo.document.update.AssignValueUpdate;
+import com.yahoo.document.update.ClearValueUpdate;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.MapValueUpdate;
+import com.yahoo.document.update.RemoveValueUpdate;
+import com.yahoo.document.update.ValueUpdate;
+import com.yahoo.vespa.objects.FieldBase;
+import com.yahoo.vespa.objects.Serializer;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+
+import static com.yahoo.document.json.JsonSerializationHelper.*;
+
+/**
+ * The DocumentUpdateJsonSerializer utility class is used to serialize a DocumentUpdate instance using the JSON format described in
+ * <a href="https://git.corp.yahoo.com/pages/vespa/documentation/documentation/reference/document-json-format.html#update">Document JSON Format: The Update Structure</a>
+ *
+ * @see #serialize(com.yahoo.document.DocumentUpdate)
+ * @author Vegard Sjonfjell
+ */
+public class DocumentUpdateJsonSerializer
+{
+    private final JsonFactory jsonFactory = new JsonFactory();
+    private final JsonDocumentUpdateWriter writer = new JsonDocumentUpdateWriter();
+    private JsonGenerator generator;
+
+    /**
+     * Instantiate a DocumentUpdateJsonSerializer that outputs JSON to an OutputStream
+     */
+    public DocumentUpdateJsonSerializer(OutputStream outputStream) {
+        wrapIOException(() -> generator = jsonFactory.createGenerator(outputStream));
+    }
+
+    /**
+     * Instantiate a DocumentUpdateJsonSerializer that writes JSON using existing JsonGenerator
+     */
+    public DocumentUpdateJsonSerializer(JsonGenerator generator) {
+        this.generator = generator;
+    }
+
+    /**
+     * Serialize a DocumentUpdate tree to JSON
+     */
+    public void serialize(DocumentUpdate update) {
+        writer.write(update);
+    }
+
+    private class JsonDocumentUpdateWriter implements DocumentUpdateWriter, FieldWriter {
+        @Override
+        public void write(DocumentUpdate update) {
+            wrapIOException(() -> {
+                generator.writeStartObject();
+                generator.writeStringField("update", update.getId().toString());
+
+                if (update.getCondition().isPresent()) {
+                    generator.writeStringField("condition", update.getCondition().getSelection());
+                }
+
+                generator.writeObjectFieldStart("fields");
+                for (FieldUpdate up : update.getFieldUpdates()) {
+                    up.serialize(this);
+                }
+                generator.writeEndObject();
+
+                generator.writeEndObject();
+                generator.flush();
+            });
+        }
+
+        @Override
+        public void write(FieldUpdate fieldUpdate) {
+            wrapIOException(() -> {
+                generator.writeObjectFieldStart(fieldUpdate.getField().getName());
+
+                ArrayList<ValueUpdate> removeValueUpdates = new ArrayList<>();
+                ArrayList<ValueUpdate> addValueUpdates = new ArrayList<>();
+
+                final DataType dataType = fieldUpdate.getField().getDataType();
+                for (ValueUpdate valueUpdate : fieldUpdate.getValueUpdates()) {
+                    if (valueUpdate instanceof RemoveValueUpdate) {
+                        removeValueUpdates.add(valueUpdate);
+                    } else if (valueUpdate instanceof AddValueUpdate) {
+                        addValueUpdates.add(valueUpdate);
+                    } else {
+                        valueUpdate.serialize(this, dataType);
+                    }
+                }
+
+                writeAddOrRemoveValueUpdates("remove", removeValueUpdates, dataType);
+                writeAddOrRemoveValueUpdates("add", addValueUpdates, dataType);
+
+                generator.writeEndObject();
+            });
+        }
+
+        private void writeAddOrRemoveValueUpdates(String arrayFieldName, ArrayList<ValueUpdate> valueUpdates, DataType dataType) throws IOException {
+            if (!valueUpdates.isEmpty()) {
+                generator.writeArrayFieldStart(arrayFieldName);
+                for (ValueUpdate valueUpdate : valueUpdates) {
+                    valueUpdate.serialize(this, dataType);
+                }
+                generator.writeEndArray();
+            }
+        }
+
+        @Override
+        public void write(AddValueUpdate update, DataType superType) {
+            update.getValue().serialize(this);
+        }
+
+        /* This is the 'match' operation */
+        @Override
+        public void write(MapValueUpdate update, DataType superType) {
+            wrapIOException(() -> {
+                generator.writeObjectFieldStart("match");
+                generator.writeFieldName("element");
+                update.getValue().serialize(null, this);
+                update.getUpdate().serialize(this, superType);
+                generator.writeEndObject();
+            });
+        }
+
+        @Override
+        public void write(ArithmeticValueUpdate update) {
+            final ArithmeticValueUpdate.Operator operator = update.getOperator();
+            final String operationKey;
+
+            switch (operator) {
+                case ADD:
+                    operationKey = "increment";
+                    break;
+                case DIV:
+                    operationKey = "divide";
+                    break;
+                case MUL:
+                    operationKey = "multiply";
+                    break;
+                case SUB:
+                    operationKey = "decrement";
+                    break;
+                default:
+                    throw new RuntimeException(String.format("Unrecognized arithmetic operator '%s'", operator.name));
+            }
+
+            wrapIOException(() -> generator.writeFieldName(operationKey));
+            update.getValue().serialize(this);
+        }
+
+        @Override
+        public void write(AssignValueUpdate update, DataType superType) {
+            wrapIOException(() -> generator.writeFieldName("assign"));
+            update.getValue().serialize(null, this);
+        }
+
+        @Override
+        public void write(RemoveValueUpdate update, DataType superType) {
+            update.getValue().serialize(null, this);
+        }
+
+        @Override
+        public void write(ClearValueUpdate clearValueUpdate, DataType superType) {
+            wrapIOException(() -> generator.writeNullField("assign"));
+        }
+
+        @Override
+        public void write(FieldBase field, FieldValue value) {
+            throw new JsonSerializationException(String.format("Serialization of field values of type %s is not supported", value.getClass().getName()));
+        }
+
+        @Override
+        public void write(FieldBase field, Document value) {
+            throw new JsonSerializationException("Serialization of 'Document fields' is not supported");
+        }
+
+        @Override
+        public <T extends FieldValue> void write(FieldBase field, Array<T> array) {
+            serializeArrayField(this, generator, field, array);
+        }
+
+        @Override
+        public <K extends FieldValue, V extends FieldValue> void write(FieldBase field, MapFieldValue<K, V> map) {
+            serializeMapField(this, generator, field, map);
+        }
+
+        @Override
+        public void write(FieldBase field, ByteFieldValue value) {
+            serializeByteField(generator, field, value);
+        }
+
+        @Override
+        public <T extends FieldValue> void write(FieldBase field, CollectionFieldValue<T> value) {
+            serializeCollectionField(this, generator, field, value);
+        }
+
+        @Override
+        public void write(FieldBase field, DoubleFieldValue value) {
+            serializeDoubleField(generator, field, value);
+        }
+
+        @Override
+        public void write(FieldBase field, FloatFieldValue value) {
+            serializeFloatField(generator, field, value);
+        }
+
+        @Override
+        public void write(FieldBase field, IntegerFieldValue value) {
+            serializeIntField(generator, field, value);
+        }
+
+        @Override
+        public void write(FieldBase field, LongFieldValue value) {
+            serializeLongField(generator, field, value);
+        }
+
+        @Override
+        public void write(FieldBase field, Raw value) {
+            serializeRawField(generator, field, value);
+        }
+
+        @Override
+        public void write(FieldBase field, PredicateFieldValue value) {
+            serializePredicateField(generator, field, value);
+        }
+
+        @Override
+        public void write(FieldBase field, StringFieldValue value) {
+            serializeStringField(generator, field, value);
+        }
+
+        @Override
+        public void write(FieldBase field, TensorFieldValue value) {
+            serializeTensorField(generator, field, value);
+        }
+
+        @Override
+        public void write(FieldBase field, Struct value) {
+            serializeStructField(this, generator, field, value);
+        }
+
+        @Override
+        public void write(FieldBase field, StructuredFieldValue value) {
+            serializeStructuredField(this, generator, field, value);
+        }
+
+        @Override
+        public <T extends FieldValue> void write(FieldBase field, WeightedSet<T> weightedSet) {
+            serializeWeightedSet(generator, field, weightedSet);
+        }
+
+        @Override
+        public void write(FieldBase field, AnnotationReference value) {
+            // Serialization of annotations are not implemented
+        }
+
+        @Override
+        public Serializer putByte(FieldBase field, byte value) {
+            serializeByte(generator, field, value);
+            return this;
+        }
+
+        @Override
+        public Serializer putShort(FieldBase field, short value) {
+            serializeShort(generator, field, value);
+            return this;
+        }
+
+        @Override
+        public Serializer putInt(FieldBase field, int value) {
+            serializeInt(generator, field, value);
+            return this;
+        }
+
+        @Override
+        public Serializer putLong(FieldBase field, long value) {
+            serializeLong(generator, field, value);
+            return this;
+        }
+
+        @Override
+        public Serializer putFloat(FieldBase field, float value) {
+            serializeFloat(generator, field, value);
+            return this;
+        }
+
+        @Override
+        public Serializer putDouble(FieldBase field, double value) {
+            serializeDouble(generator, field, value);
+            return this;
+        }
+
+        @Override
+        public Serializer put(FieldBase field, byte[] value) {
+            serializeByteArray(generator, field, value);
+            return this;
+        }
+
+        @Override
+        public Serializer put(FieldBase field, ByteBuffer value) {
+            serializeByteBuffer(generator, field, value);
+            return this;
+        }
+
+        @Override
+        public Serializer put(FieldBase field, String value) {
+            serializeString(generator, field, value);
+            return this;
+        }
+    }
+}
\ No newline at end of file
diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java
new file mode 100644
index 00000000000..639c6ad5300
--- /dev/null
+++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java
@@ -0,0 +1,297 @@
+package com.yahoo.document.json;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.yahoo.document.Field;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.ByteFieldValue;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.TensorAddress;
+import com.yahoo.vespa.objects.FieldBase;
+import org.apache.commons.codec.binary.Base64;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author Steinar Knutsen
+ * @author Vegard Sjonfjell
+ */
+public class JsonSerializationHelper {
+    private final static Base64 base64Encoder = new Base64();
+
+    static class JsonSerializationException extends RuntimeException {
+        public JsonSerializationException(Exception base) {
+            super(base);
+        }
+
+        public JsonSerializationException(String message) {
+            super(message);
+        }
+    }
+
+    @FunctionalInterface
+    static interface SubroutineThrowingIOException {
+        void invoke() throws IOException;
+    }
+
+    static void wrapIOException(SubroutineThrowingIOException lambda) {
+        try {
+            lambda.invoke();
+        } catch (IOException e) {
+            throw new JsonSerializationException(e);
+        }
+    }
+
+    public static void serializeTensorField(JsonGenerator generator, FieldBase field, TensorFieldValue value) {
+        wrapIOException(() -> {
+            fieldNameIfNotNull(generator, field);
+            generator.writeStartObject();
+
+            if (value.getTensor().isPresent()) {
+                Tensor tensor = value.getTensor().get();
+                serializeTensorDimensions(generator, tensor.dimensions());
+                serializeTensorCells(generator, tensor.cells());
+            }
+            generator.writeEndObject();
+        });
+    }
+
+    private static void serializeTensorDimensions(JsonGenerator generator, Set<String> dimensions) throws IOException {
+        generator.writeArrayFieldStart(JsonReader.TENSOR_DIMENSIONS);
+        for (String dimension : dimensions) {
+            generator.writeString(dimension);
+        }
+
+        generator.writeEndArray();
+    }
+
+    private static void serializeTensorCells(JsonGenerator generator, Map<TensorAddress, Double> cells) throws IOException {
+        generator.writeArrayFieldStart(JsonReader.TENSOR_CELLS);
+        for (Map.Entry<TensorAddress, Double> cell : cells.entrySet()) {
+            generator.writeStartObject();
+            serializeTensorAddress(generator, cell.getKey());
+            generator.writeNumberField(JsonReader.TENSOR_VALUE, cell.getValue());
+            generator.writeEndObject();
+        }
+
+        generator.writeEndArray();
+    }
+
+    private static void serializeTensorAddress(JsonGenerator generator, TensorAddress address) throws IOException {
+        generator.writeObjectFieldStart(JsonReader.TENSOR_ADDRESS);
+        for (TensorAddress.Element element : address.elements()) {
+            generator.writeStringField(element.dimension(), element.label());
+        }
+
+        generator.writeEndObject();
+    }
+
+
+    public static void serializeStringField(JsonGenerator generator, FieldBase field, StringFieldValue value) {
+        // Hide empty strings
+        if (value.getString().length() == 0) {
+            return;
+        }
+
+        serializeString(generator, field, value.getString());
+    }
+
+    public static void serializeStructuredField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, StructuredFieldValue value) {
+        fieldNameIfNotNull(generator, field);
+
+        wrapIOException(() -> {
+            generator.writeStartObject();
+            Iterator<Map.Entry<Field, FieldValue>> i = value.iterator();
+
+            while (i.hasNext()) {
+                Map.Entry<Field, FieldValue> entry = i.next();
+                entry.getValue().serialize(entry.getKey(), fieldWriter);
+            }
+
+            generator.writeEndObject();
+        });
+    }
+
+    public static void serializeStructField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, Struct value) {
+        if (value.getDataType() == PositionDataType.INSTANCE) {
+            serializeString(generator, field, PositionDataType.renderAsString(value));
+            return;
+        }
+
+        serializeStructuredField(fieldWriter, generator, field, value);
+    }
+
+    public static <T extends FieldValue> void serializeWeightedSet(JsonGenerator generator, FieldBase field, WeightedSet<T> value) {
+        fieldNameIfNotNull(generator, field);
+
+        wrapIOException(() -> {
+            generator.writeStartObject();
+
+            for (T key : value.keySet()) {
+                Integer weight = value.get(key);
+                // key.toString() is according to spec
+                generator.writeNumberField(key.toString(), weight);
+            }
+
+            generator.writeEndObject();
+        });
+    }
+
+    public static <T extends FieldValue> void serializeCollectionField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, CollectionFieldValue<T> value) {
+        fieldNameIfNotNull(generator, field);
+
+        wrapIOException(() -> {
+            generator.writeStartArray();
+            Iterator<T> i = value.iterator();
+
+            while (i.hasNext()) {
+                i.next().serialize(null, fieldWriter);
+            }
+
+            generator.writeEndArray();
+        });
+    }
+
+
+    public static <K extends FieldValue, V extends FieldValue> void serializeMapField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, MapFieldValue<K, V> map) {
+        fieldNameIfNotNull(generator, field);
+        wrapIOException(() -> {
+            generator.writeStartArray();
+
+            for (Map.Entry<K, V> entry : map.entrySet()) {
+                generator.writeStartObject();
+                generator.writeFieldName(JsonReader.MAP_KEY);
+                entry.getKey().serialize(null, fieldWriter);
+                generator.writeFieldName(JsonReader.MAP_VALUE);
+                entry.getValue().serialize(null, fieldWriter);
+                generator.writeEndObject();
+            }
+
+            generator.writeEndArray();
+        });
+    }
+
+    public static <T extends FieldValue> void serializeArrayField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, Array<T> value) {
+        wrapIOException(() -> {
+            fieldNameIfNotNull(generator, field);
+            generator.writeStartArray();
+
+            for (T elem : value) {
+                elem.serialize(null, fieldWriter);
+            }
+
+            generator.writeEndArray();
+        });
+    }
+
+    public static void serializeDoubleField(JsonGenerator generator, FieldBase field, DoubleFieldValue value) {
+        serializeDouble(generator, field, value.getDouble());
+    }
+
+    public static void serializeFloatField(JsonGenerator generator, FieldBase field, FloatFieldValue value) {
+        serializeFloat(generator, field, value.getFloat());
+    }
+
+    public static void serializeIntField(JsonGenerator generator, FieldBase field, IntegerFieldValue value) {
+        serializeInt(generator, field, value.getInteger());
+    }
+
+    public static void serializeLongField(JsonGenerator generator, FieldBase field, LongFieldValue value) {
+        serializeLong(generator, field, value.getLong());
+    }
+
+    public static void serializeByteField(JsonGenerator generator, FieldBase field, ByteFieldValue value) {
+        serializeByte(generator, field, value.getByte());
+    }
+
+    public static void serializePredicateField(JsonGenerator generator, FieldBase field, PredicateFieldValue value){
+        serializeString(generator, field, value.toString());
+    }
+
+    public static void serializeRawField(JsonGenerator generator, FieldBase field, Raw raw) {
+        serializeByteBuffer(generator, field, raw.getByteBuffer());
+    }
+
+    public static void serializeString(JsonGenerator generator, FieldBase field, String value) {
+        if (value.length() == 0) {
+            return;
+        }
+
+        fieldNameIfNotNull(generator, field);
+        wrapIOException(() -> generator.writeString(value));
+    }
+
+    public static void serializeByte(JsonGenerator generator, FieldBase field,  byte value) {
+        fieldNameIfNotNull(generator, field);
+        wrapIOException(() -> generator.writeNumber(value));
+    }
+
+    public static void serializeShort(JsonGenerator generator, FieldBase field, short value) {
+        fieldNameIfNotNull(generator, field);
+        wrapIOException(() -> generator.writeNumber(value));
+    }
+
+    public static void serializeInt(JsonGenerator generator, FieldBase field, int value) {
+        fieldNameIfNotNull(generator, field);
+        wrapIOException(() -> generator.writeNumber(value));
+    }
+
+    public static void serializeLong(JsonGenerator generator, FieldBase field, long value) {
+        fieldNameIfNotNull(generator, field);
+        wrapIOException(() -> generator.writeNumber(value));
+    }
+
+    public static void serializeFloat(JsonGenerator generator, FieldBase field, float value) {
+        fieldNameIfNotNull(generator, field);
+        wrapIOException(() -> generator.writeNumber(value));
+    }
+
+    public static void serializeDouble(JsonGenerator generator, FieldBase field, double value) {
+        fieldNameIfNotNull(generator, field);
+        wrapIOException(() -> generator.writeNumber(value));
+    }
+
+    public static void serializeByteBuffer(JsonGenerator generator, FieldBase field, ByteBuffer raw) {
+        fieldNameIfNotNull(generator, field);
+
+        final byte[] data = new byte[raw.remaining()];
+        final int origPosition = raw.position();
+
+        // base64encoder has no encode methods with offset and
+        // limit, so no use trying to get at the backing array if
+        // available anyway
+        raw.get(data);
+        raw.position(origPosition);
+
+        wrapIOException(() -> generator.writeString(base64Encoder.encodeToString(data)));
+    }
+
+    public static void serializeByteArray(JsonGenerator generator, FieldBase field, byte[] value) {
+        serializeByteBuffer(generator, field, ByteBuffer.wrap(value));
+    }
+
+    public static void fieldNameIfNotNull(JsonGenerator generator, FieldBase field) {
+        if (field != null) {
+            wrapIOException(() -> generator.writeFieldName(field.getName()));
+        }
+    }
+}
diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java
index 79a1c040cbc..626c97a958a 100644
--- a/document/src/main/java/com/yahoo/document/json/JsonWriter.java
+++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java
@@ -1,34 +1,43 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.document.json;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import com.yahoo.document.datatypes.*;
-import com.yahoo.tensor.Tensor;
-import com.yahoo.tensor.TensorAddress;
-import org.apache.commons.codec.binary.Base64;
-
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.yahoo.document.Document;
 import com.yahoo.document.DocumentId;
 import com.yahoo.document.DocumentType;
 import com.yahoo.document.Field;
-import com.yahoo.document.PositionDataType;
 import com.yahoo.document.annotation.AnnotationReference;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.ByteFieldValue;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
 import com.yahoo.document.serialization.DocumentWriter;
 import com.yahoo.vespa.objects.FieldBase;
 import com.yahoo.vespa.objects.Serializer;
-
 import edu.umd.cs.findbugs.annotations.NonNull;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.Map;
+
+import static com.yahoo.document.json.JsonSerializationHelper.*;
+
 /**
  * Serialize Document and other FieldValue instances as JSON.
  *
@@ -38,7 +47,6 @@ public class JsonWriter implements DocumentWriter {
 
     private static final JsonFactory jsonFactory = new JsonFactory();
     private final JsonGenerator generator;
-    private final Base64 base64Encoder = new Base64();
 
     // I really hate exception unsafe constructors, but the alternative
     // requires generator to not be a final
@@ -85,22 +93,23 @@ public class JsonWriter implements DocumentWriter {
      */
     @Override
     public void write(FieldBase field, FieldValue value) {
-        throw new UnsupportedOperationException("Serializing "
-                + value.getClass().getName() + " is not supported.");
+        throw new UnsupportedOperationException("Serializing " + value.getClass().getName() + " is not supported.");
     }
 
     @Override
     public void write(FieldBase field, Document value) {
         try {
-            fieldNameIfNotNull(field);
+            fieldNameIfNotNull(generator, field);
             generator.writeStartObject();
+
             // this makes it impossible to refeed directly, not sure what's correct
             // perhaps just change to "put"?
             generator.writeStringField("id", value.getId().toString());
             generator.writeObjectFieldStart(JsonReader.FIELDS);
-            for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i
-                    .hasNext();) {
-                Entry<Field, FieldValue> entry = i.next();
+
+            Iterator<Map.Entry<Field, FieldValue>> i = value.iterator();
+            while (i.hasNext()) {
+                Map.Entry<Field, FieldValue> entry = i.next();
                 entry.getValue().serialize(entry.getKey(), this);
             }
             generator.writeEndObject();
@@ -113,199 +122,77 @@ public class JsonWriter implements DocumentWriter {
 
     @Override
     public <T extends FieldValue> void write(FieldBase field, Array<T> value) {
-        try {
-            fieldNameIfNotNull(field);
-            generator.writeStartArray();
-            for (Iterator<T> i = value.iterator(); i.hasNext();) {
-                i.next().serialize(null, this);
-            }
-            generator.writeEndArray();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-
-    }
-
-    private void fieldNameIfNotNull(FieldBase field) {
-        if (field != null) {
-            try {
-                generator.writeFieldName(field.getName());
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        }
+        serializeArrayField(this, generator, field, value);
     }
 
     @Override
-    public <K extends FieldValue, V extends FieldValue> void write(
-            FieldBase field, MapFieldValue<K, V> map) {
-        fieldNameIfNotNull(field);
-        try {
-            generator.writeStartArray();
-            for (Map.Entry<K, V> entry : map.entrySet()) {
-                generator.writeStartObject();
-                generator.writeFieldName(JsonReader.MAP_KEY);
-                entry.getKey().serialize(null, this);
-                generator.writeFieldName(JsonReader.MAP_VALUE);
-                entry.getValue().serialize(null, this);
-                generator.writeEndObject();
-            }
-            generator.writeEndArray();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
+    public <K extends FieldValue, V extends FieldValue> void write(FieldBase field, MapFieldValue<K, V> map) {
+        serializeMapField(this, generator, field, map);
     }
 
     @Override
     public void write(FieldBase field, ByteFieldValue value) {
-        putByte(field, value.getByte());
+        serializeByteField(generator, field, value);
     }
 
     @Override
-    public <T extends FieldValue> void write(FieldBase field,
-            CollectionFieldValue<T> value) {
-        fieldNameIfNotNull(field);
-        try {
-            generator.writeStartArray();
-            for (Iterator<T> i = value.iterator(); i.hasNext();) {
-                i.next().serialize(null, this);
-            }
-            generator.writeEndArray();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
+    public <T extends FieldValue> void write(FieldBase field, CollectionFieldValue<T> value) {
+        serializeCollectionField(this, generator, field, value);
     }
 
     @Override
     public void write(FieldBase field, DoubleFieldValue value) {
-        putDouble(field, value.getDouble());
+        serializeDoubleField(generator, field, value);
     }
 
     @Override
     public void write(FieldBase field, FloatFieldValue value) {
-        putFloat(field, value.getFloat());
+        serializeFloatField(generator, field, value);
     }
 
     @Override
     public void write(FieldBase field, IntegerFieldValue value) {
-        putInt(field, value.getInteger());
+        serializeIntField(generator, field, value);
     }
 
     @Override
     public void write(FieldBase field, LongFieldValue value) {
-        putLong(field, value.getLong());
+        serializeLongField(generator, field, value);
     }
 
     @Override
     public void write(FieldBase field, Raw value) {
-        put(field, value.getByteBuffer());
+        serializeRawField(generator, field, value);
     }
 
     @Override
     public void write(FieldBase field, PredicateFieldValue value) {
-        put(field, value.toString());
+        serializePredicateField(generator, field, value);
     }
 
     @Override
     public void write(FieldBase field, StringFieldValue value) {
-        put(field, value.getString());
+        serializeStringField(generator, field, value);
     }
 
     @Override
     public void write(FieldBase field, TensorFieldValue value) {
-        try {
-            fieldNameIfNotNull(field);
-            generator.writeStartObject();
-            if (value.getTensor().isPresent()) {
-                Tensor tensor = value.getTensor().get();
-                writeTensorDimensions(tensor.dimensions());
-                writeTensorCells(tensor.cells());
-            }
-            generator.writeEndObject();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    private void writeTensorDimensions(Set<String> dimensions) throws IOException {
-        generator.writeArrayFieldStart(JsonReader.TENSOR_DIMENSIONS);
-        for (String dimension : dimensions) {
-            generator.writeString(dimension);
-        }
-        generator.writeEndArray();
-    }
-
-    private void writeTensorCells(Map<TensorAddress, Double> cells) throws IOException {
-        generator.writeArrayFieldStart(JsonReader.TENSOR_CELLS);
-        for (Map.Entry<TensorAddress, Double> cell : cells.entrySet()) {
-            generator.writeStartObject();
-            writeTensorAddress(cell.getKey());
-            generator.writeNumberField(JsonReader.TENSOR_VALUE, cell.getValue());
-            generator.writeEndObject();
-        }
-        generator.writeEndArray();
-    }
-
-    private void writeTensorAddress(TensorAddress address) throws IOException {
-        generator.writeObjectFieldStart(JsonReader.TENSOR_ADDRESS);
-        for (TensorAddress.Element element : address.elements()) {
-            generator.writeStringField(element.dimension(), element.label());
-        }
-        generator.writeEndObject();
+        serializeTensorField(generator, field, value);
     }
 
     @Override
     public void write(FieldBase field, Struct value) {
-        if (value.getDataType() == PositionDataType.INSTANCE) {
-            put(field, PositionDataType.renderAsString(value));
-            return;
-        }
-        fieldNameIfNotNull(field);
-        try {
-            generator.writeStartObject();
-            for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i
-                    .hasNext();) {
-                Entry<Field, FieldValue> entry = i.next();
-                entry.getValue().serialize(entry.getKey(), this);
-            }
-            generator.writeEndObject();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
+        serializeStructField(this, generator, field, value);
     }
 
     @Override
     public void write(FieldBase field, StructuredFieldValue value) {
-        fieldNameIfNotNull(field);
-        try {
-            generator.writeStartObject();
-            for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i
-                    .hasNext();) {
-                Entry<Field, FieldValue> entry = i.next();
-                entry.getValue().serialize(entry.getKey(), this);
-            }
-            generator.writeEndObject();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
+        serializeStructuredField(this, generator, field, value);
     }
 
     @Override
-    public <T extends FieldValue> void write(FieldBase field,
-            WeightedSet<T> value) {
-        fieldNameIfNotNull(field);
-        try {
-            generator.writeStartObject();
-            // entrySet() is deprecated and there is no entry iterator
-            for (T key : value.keySet()) {
-                Integer weight = value.get(key);
-                // key.toString() is according to spec
-                generator.writeNumberField(key.toString(), weight);
-            }
-            generator.writeEndObject();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
+    public <T extends FieldValue> void write(FieldBase field, WeightedSet<T> value) {
+        serializeWeightedSet(generator, field, value);
     }
 
     @Override
@@ -315,110 +202,6 @@ public class JsonWriter implements DocumentWriter {
     }
 
     @Override
-    public Serializer putByte(FieldBase field, byte value) {
-        fieldNameIfNotNull(field);
-        try {
-            generator.writeNumber(value);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        return this;
-    }
-
-    @Override
-    public Serializer putShort(FieldBase field, short value) {
-        fieldNameIfNotNull(field);
-        try {
-            generator.writeNumber(value);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        return this;
-    }
-
-    @Override
-    public Serializer putInt(FieldBase field, int value) {
-        fieldNameIfNotNull(field);
-        try {
-            generator.writeNumber(value);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        return this;
-    }
-
-    @Override
-    public Serializer putLong(FieldBase field, long value) {
-        fieldNameIfNotNull(field);
-        try {
-            generator.writeNumber(value);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        return this;
-    }
-
-    @Override
-    public Serializer putFloat(FieldBase field, float value) {
-        fieldNameIfNotNull(field);
-        try {
-            generator.writeNumber(value);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        return this;
-    }
-
-    @Override
-    public Serializer putDouble(FieldBase field, double value) {
-        fieldNameIfNotNull(field);
-        try {
-            generator.writeNumber(value);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        return this;
-    }
-
-    @Override
-    public Serializer put(FieldBase field, byte[] value) {
-        return put(field, ByteBuffer.wrap(value));
-    }
-
-    @Override
-    public Serializer put(FieldBase field, ByteBuffer raw) {
-        final byte[] data = new byte[raw.remaining()];
-        final int origPosition = raw.position();
-
-        fieldNameIfNotNull(field);
-        // base64encoder has no encode methods with offset and
-        // limit, so no use trying to get at the backing array if
-        // available anyway
-        raw.get(data);
-        raw.position(origPosition);
-        try {
-            generator.writeString(base64Encoder.encodeToString(data));
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        return this;
-    }
-
-    @Override
-    public Serializer put(FieldBase field, String value) {
-        if (value.length() == 0) {
-            return this;
-        }
-        fieldNameIfNotNull(field);
-        try {
-            generator.writeString(value);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        return this;
-    }
-
-    @Override
     public void write(Document document) {
         write(null, document);
     }
@@ -470,4 +253,58 @@ public class JsonWriter implements DocumentWriter {
         }
         return out.toByteArray();
     }
+
+    @Override
+    public Serializer putByte(FieldBase field, byte value) {
+        serializeByte(generator, field, value);
+        return this;
+    }
+
+    @Override
+    public Serializer putShort(FieldBase field, short value) {
+        serializeShort(generator, field, value);
+        return this;
+    }
+
+    @Override
+    public Serializer putInt(FieldBase field, int value) {
+        serializeInt(generator, field, value);
+        return this;
+    }
+
+    @Override
+    public Serializer putLong(FieldBase field, long value) {
+        serializeLong(generator, field, value);
+        return this;
+    }
+
+    @Override
+    public Serializer putFloat(FieldBase field, float value) {
+        serializeFloat(generator, field, value);
+        return this;
+    }
+
+    @Override
+    public Serializer putDouble(FieldBase field, double value) {
+        serializeDouble(generator, field, value);
+        return this;
+    }
+
+    @Override
+    public Serializer put(FieldBase field, byte[] value) {
+        serializeByteArray(generator, field, value);
+        return this;
+    }
+
+    @Override
+    public Serializer put(FieldBase field, ByteBuffer value) {
+        serializeByteBuffer(generator, field, value);
+        return this;
+    }
+
+    @Override
+    public Serializer put(FieldBase field, String value) {
+        serializeString(generator, field, value);
+        return this;
+    }
 }
diff --git a/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java b/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java
index 4a269a704d2..244aa0d5a0d 100644
--- a/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java
+++ b/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java
@@ -3,7 +3,22 @@ package com.yahoo.document.serialization;
 
 import com.yahoo.document.Document;
 import com.yahoo.document.annotation.AnnotationReference;
-import com.yahoo.document.datatypes.*;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.ByteFieldValue;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
 import com.yahoo.vespa.objects.FieldBase;
 import com.yahoo.vespa.objects.Serializer;
 
@@ -189,5 +204,4 @@ public interface FieldWriter extends Serializer {
      *            field value
      */
     void write(FieldBase field, AnnotationReference value);
-
 }
diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java
index ffe6073cd6c..dc605fbe3d5 100644
--- a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java
+++ b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java
@@ -1,21 +1,36 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.document.serialization;
 
-import java.nio.ByteBuffer;
-import java.util.ArrayDeque;
-import java.util.Deque;
-import java.util.Iterator;
-import java.util.Map;
-
 import com.yahoo.document.Document;
 import com.yahoo.document.DocumentId;
 import com.yahoo.document.DocumentType;
 import com.yahoo.document.Field;
 import com.yahoo.document.annotation.AnnotationReference;
-import com.yahoo.document.datatypes.*;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.ByteFieldValue;
+import com.yahoo.document.datatypes.CollectionFieldValue;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
 import com.yahoo.vespa.objects.FieldBase;
 import com.yahoo.vespa.objects.Serializer;
 
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.Map;
+
 // TODO: Just inline all use of XmlSerializationHelper when the toXml methods in FieldValue subclasses are to be removed
 // TODO: More cleanup, the put() methods generate a lot of superfluous objects (write should call put, not the other way around)
 // TODO: remove pingpong between XmlSerializationHelper and FieldValue, this will go away when the toXml methods go away