Publish

103 files changed, 19690 insertions, 0 deletions

diff --git a/document/src/test/java/com/yahoo/document/BucketIdFactoryTestCase.java b/document/src/test/java/com/yahoo/document/BucketIdFactoryTestCase.java
new file mode 100644
index 00000000000..e511e4cf378
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/BucketIdFactoryTestCase.java
@@ -0,0 +1,130 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.BucketId;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.idstring.*;
+
+/**
+ * Date: Sep 7, 2007
+ *
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class BucketIdFactoryTestCase extends junit.framework.TestCase {
+
+    public BucketIdFactoryTestCase(String name) {
+        super(name);
+    }
+
+    public void testNormalUsage() {
+        BucketIdFactory factory = new BucketIdFactory();
+        assertEquals(BucketId.COUNT_BITS, factory.getCountBitCount());
+
+        assertEquals(64, factory.getLocationBitCount() + factory.getGidBitCount() + BucketId.COUNT_BITS);
+    }
+
+    private class Hex {
+        long value;
+
+        Hex(long val) {
+            value = val;
+        }
+
+        public boolean equals(Object o) {
+            return (o instanceof Hex && value == ((Hex) o).value);
+        }
+
+        public String toString() {
+            return Long.toHexString(value);
+        }
+    }
+
+    public void testBucketGeneration() {
+        BucketIdFactory factory = new BucketIdFactory(32, 26, 6);
+        DocumentId doc1 = new DocumentId(new DocIdString("ns", "spec"));
+        DocumentId doc2 = new DocumentId(new DocIdString("ns2", "spec"));
+        DocumentId doc3 = new DocumentId(new DocIdString("ns", "spec2"));
+        DocumentId userDoc1 = new DocumentId(new UserDocIdString("ns", 0x12, "spec"));
+        DocumentId userDoc2 = new DocumentId(new UserDocIdString("ns2", 0x12, "spec2"));
+        DocumentId userDoc3 = new DocumentId(new UserDocIdString("ns", 0x13, "spec"));
+        DocumentId groupDoc1 = new DocumentId(new GroupDocIdString("ns", "yahoo.com", "spec"));
+        DocumentId groupDoc2 = new DocumentId(new GroupDocIdString("ns2", "yahoo.com", "spec2"));
+        DocumentId groupDoc3 = new DocumentId(new GroupDocIdString("ns", "yahoo", "spec"));
+        DocumentId orderDoc1 = new DocumentId(new OrderDocIdString("ns", "13", 31, 19, 1268182861, "foo"));
+        DocumentId orderDoc2 = new DocumentId(new OrderDocIdString("ns", "13", 31, 19, 1205110861, "foo"));
+        DocumentId orderDoc3 = new DocumentId(new OrderDocIdString("ns", "13", 31, 19, 1205715661, "foo"));
+        DocumentId orderDoc4 = new DocumentId(new OrderDocIdString("ns", "13", 4, 0, 2, "foo"));
+        DocumentId orderDoc5 = new DocumentId(new OrderDocIdString("ns", "13", 4, 0, 4, "foo"));
+        DocumentId orderDoc6 = new DocumentId(new OrderDocIdString("ns", "13", 4, 0, 11, "foo"));
+
+        BucketId docBucket1 = factory.getBucketId(doc1);
+        BucketId docBucket2 = factory.getBucketId(doc2);
+        BucketId docBucket3 = factory.getBucketId(doc3);
+        BucketId userDocBucket1 = factory.getBucketId(userDoc1);
+        BucketId userDocBucket2 = factory.getBucketId(userDoc2);
+        BucketId userDocBucket3 = factory.getBucketId(userDoc3);
+        BucketId groupDocBucket1 = factory.getBucketId(groupDoc1);
+        BucketId groupDocBucket2 = factory.getBucketId(groupDoc2);
+        BucketId groupDocBucket3 = factory.getBucketId(groupDoc3);
+        BucketId orderDocBucket1 = factory.getBucketId(orderDoc1);
+        BucketId orderDocBucket2 = factory.getBucketId(orderDoc2);
+        BucketId orderDocBucket3 = factory.getBucketId(orderDoc3);
+        BucketId orderDocBucket4 = factory.getBucketId(orderDoc4);
+        BucketId orderDocBucket5 = factory.getBucketId(orderDoc5);
+        BucketId orderDocBucket6 = factory.getBucketId(orderDoc6);
+
+        assertEquals(new Hex(0xe99703f200000012l), new Hex(userDocBucket1.getRawId()));
+        assertEquals(new Hex(0xebfa518a00000012l), new Hex(userDocBucket2.getRawId()));
+        assertEquals(new Hex(0xeac1850800000013l), new Hex(userDocBucket3.getRawId()));
+
+        assertEquals(new Hex(0xe90ce4b09a1acd50l), new Hex(groupDocBucket1.getRawId()));
+        assertEquals(new Hex(0xe9cedaa49a1acd50l), new Hex(groupDocBucket2.getRawId()));
+        assertEquals(new Hex(0xe8cdb18bafe81f24l), new Hex(groupDocBucket3.getRawId()));
+
+        assertEquals(new Hex(0xe980c9abd5fd8d11l), new Hex(docBucket1.getRawId()));
+        assertEquals(new Hex(0xeafe870c5f9c37b9l), new Hex(docBucket2.getRawId()));
+        assertEquals(new Hex(0xeaebe9473ecbcd69l), new Hex(docBucket3.getRawId()));
+
+        assertEquals(new Hex(0xeae764e90000000dl), new Hex(orderDocBucket1.getRawId()));
+        assertEquals(new Hex(0xeacb85f10000000dl), new Hex(orderDocBucket2.getRawId()));
+        assertEquals(new Hex(0xea68ddf10000000dl), new Hex(orderDocBucket3.getRawId()));
+
+        assertEquals(new Hex(0xe87526540000000dl), new Hex(orderDocBucket4.getRawId()));
+        assertEquals(new Hex(0xea59f8f20000000dl), new Hex(orderDocBucket5.getRawId()));
+        assertEquals(new Hex(0xe9eb703d0000000dl), new Hex(orderDocBucket6.getRawId()));
+    }
+
+    //Actually a BucketId testcase ...
+    public void testBidContainsBid() {
+        BucketId bid = new BucketId(18, 0x123456789L);
+
+        assert(bid.contains(new BucketId(20, 0x123456789L)));
+        assert(bid.contains(new BucketId(18, 0x888f56789L)));
+        assert(bid.contains(new BucketId(24, 0x888456789L)));
+        assert(!bid.contains(new BucketId(24, 0x888886789L)));
+        assert(!bid.contains(new BucketId(16, 0x123456789L)));
+    }
+
+    public void testBidContainsDocId() {
+        DocumentId docId = new DocumentId("userdoc:recovery:18:99999");
+        BucketIdFactory factory = new BucketIdFactory(32, 26, 6);
+        BucketId bid = new BucketId(16, 0x12L);
+        assert(bid.contains(docId, factory));
+        //split on '0'
+        bid = new BucketId(17, 0x12L);
+        assert(bid.contains(docId, factory));
+        //split on '1'
+        bid = new BucketId(17, (1L<<16) + 0x12L);
+        assert(!bid.contains(docId, factory));
+    }
+
+    public void testBucketIdSerializationAndCompare() {
+        BucketId bid = new BucketId(18, 0x123456789L);
+
+        assertEquals(bid, new BucketId(bid.toString()));
+        assertEquals(0, bid.compareTo(new BucketId(18, 0x123456789L)));
+    }
+
+}
+
diff --git a/document/src/test/java/com/yahoo/document/DataTypeNameTestCase.java b/document/src/test/java/com/yahoo/document/DataTypeNameTestCase.java
new file mode 100644
index 00000000000..c7d9ac93592
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DataTypeNameTestCase.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DataTypeNameTestCase {
+
+    @Test
+    @SuppressWarnings("deprecation")
+    public void requireThatAccessorsWork() {
+        DataTypeName name = new DataTypeName("foo");
+        assertEquals("foo", name.getName());
+        assertEquals("foo", name.toString());
+        assertFalse(name.equals(new Object()));
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/DataTypeTestCase.java b/document/src/test/java/com/yahoo/document/DataTypeTestCase.java
new file mode 100644
index 00000000000..89f22e97d5d
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DataTypeTestCase.java
@@ -0,0 +1,146 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.yahoo.document.datatypes.*;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class DataTypeTestCase {
+
+    @Test
+    public void testWeightedSetTypes() {
+        DataType stringDefault = DataType.getWeightedSet(DataType.STRING);
+        DataType stringTag=DataType.getWeightedSet(DataType.STRING,true,true);
+        assertTrue(stringDefault.equals(stringDefault));
+        assertTrue(stringTag.equals(stringTag));
+        assertFalse(stringDefault.equals(stringTag));
+        assertEquals("WeightedSet<string>",stringDefault.getName());
+        assertEquals(18, stringTag.getCode());
+        //assertEquals("WeightedSet<string>;Add;Remove",stringTag.getName());
+        assertEquals("tag",stringTag.getName());
+    }
+
+    @Test
+    public void testTagType() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        //assertEquals(DataType.getWeightedSet(DataType.STRING,true,true),DataType.TAG.getBaseType());
+        assertNotNull(manager.getDataType("tag"));
+        assertEquals(DataType.TAG, manager.getDataType("tag"));
+        assertEquals(manager.getDataType("tag").getCode(), 18);
+        assertEquals(DataType.getWeightedSet(DataType.STRING,true,true).getCode(), 18);
+    }
+
+    @Test
+    public void requireThatPredicateDataTypeIsNamedPredicate() {
+        assertEquals("predicate", DataType.PREDICATE.getName());
+    }
+
+    @Test
+    public void requireThatPredicateDataTypeHasId20() {
+        assertEquals(20, DataType.PREDICATE.getId());
+    }
+
+    @Test
+    public void requireThatPredicateDataTypeYieldsPredicateFieldValue() {
+        assertEquals(PredicateFieldValue.class, DataType.PREDICATE.createFieldValue().getClass());
+    }
+
+    @Test
+    public void testCreateFieldValueWithArg() {
+        {
+            ByteFieldValue bfv = (ByteFieldValue) DataType.BYTE.createFieldValue((byte) 4);
+            assertEquals((byte) 4, bfv.getByte());
+        }
+        {
+            ByteFieldValue bfv2 = (ByteFieldValue) DataType.BYTE.createFieldValue(4);
+            assertEquals((byte) 4, bfv2.getByte());
+        }
+        {
+            DoubleFieldValue dfv = (DoubleFieldValue) DataType.DOUBLE.createFieldValue(5.5d);
+            assertEquals(5.5d, dfv.getDouble(), 1E-6);
+        }
+        {
+            FloatFieldValue ffv = (FloatFieldValue) DataType.FLOAT.createFieldValue(5.5f);
+            assertEquals(5.5f, ffv.getFloat(), 1E-6);
+        }
+        {
+            IntegerFieldValue ifv = (IntegerFieldValue) DataType.INT.createFieldValue(5);
+            assertEquals(5, ifv.getInteger());
+        }
+        {
+            LongFieldValue lfv = (LongFieldValue) DataType.LONG.createFieldValue(34L);
+            assertEquals(34L, lfv.getLong());
+        }
+        {
+            StringFieldValue sfv = (StringFieldValue) DataType.STRING.createFieldValue("foo");
+            assertEquals("foo", sfv.getString());
+        }
+        // TODO: the 3 following should be made to not throw a silent ReflectiveOperationException in createFieldValue
+        {
+            Map<String, Integer> wsetMap = new LinkedHashMap<>();
+            wsetMap.put("foo", 1);
+            WeightedSet<StringFieldValue> ws = (WeightedSet<StringFieldValue>) DataType.getWeightedSet(DataType.STRING).createFieldValue(wsetMap);
+            assertEquals(ws.get(new StringFieldValue("foo")), new Integer(1));
+        }
+        {
+            List<String> arrayArray = new ArrayList<>();
+            arrayArray.add("foo");
+            Array<StringFieldValue> array = (Array<StringFieldValue>) DataType.getArray(DataType.STRING).createFieldValue(arrayArray);
+            assertEquals(array.get(0), new StringFieldValue("foo"));
+        }
+        {
+            Map<String, String> mapMap = new LinkedHashMap<>();
+            mapMap.put("foo", "bar");
+            MapFieldValue<StringFieldValue, StringFieldValue> map = (MapFieldValue<StringFieldValue, StringFieldValue>) DataType.getMap(DataType.STRING, DataType.STRING).createFieldValue(mapMap);
+            assertEquals(map.get(new StringFieldValue("foo")), new StringFieldValue("bar"));
+        }
+    }
+
+    @Test
+    public void testIllegalFieldPathsInArrayDataType() {
+        ArrayDataType adt = DataType.getArray(DataType.STRING);
+        try {
+            adt.buildFieldPath("[      ");
+            fail("Should have gotten exception for illegal field path.");
+        } catch (IllegalArgumentException iae) {
+            // ok!
+        }
+    }
+
+    @Test
+    public void testCloningArrayDataType() {
+        ArrayDataType adt = DataType.getArray(DataType.STRING);
+        ArrayDataType adtClone = adt.clone();
+
+        assertNotSame(adt, adtClone);
+        assertEquals(adt, adtClone);
+        assertEquals(adt.getNestedType(), adtClone.getNestedType());
+        //we should consider NOT cloning primitive types, but they are mutable and just ugly, so just clone them
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void testInstantiatingArray() {
+        ArrayDataType adt = DataType.getArray(DataType.STRING);
+        Array<StringFieldValue> val = adt.createFieldValue();
+        val.add(new StringFieldValue("foobar"));
+        assertEquals(1, val.size());
+    }
+
+    @Test
+    public void requireThatCompareToIsImplemented() {
+        assertEquals(0, DataType.INT.compareTo(DataType.INT));
+        assertTrue(DataType.INT.compareTo(DataType.STRING) < 0);
+        assertTrue(DataType.STRING.compareTo(DataType.INT) > 0);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocInDocTestCase.java b/document/src/test/java/com/yahoo/document/DocInDocTestCase.java
new file mode 100644
index 00000000000..ec6e287d105
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocInDocTestCase.java
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import junit.framework.TestCase;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotSame;
+import static junit.framework.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DocInDocTestCase extends TestCase {
+
+    @Test
+    public void testDocInDoc() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/java/com/yahoo/document/documentmanager.docindoc.cfg");
+
+        Document inner1 = new Document(manager.getDocumentType("docindoc"), "doc:inner:number:one");
+        inner1.setFieldValue("name", new StringFieldValue("Donald Duck"));
+        inner1.setFieldValue("content", new StringFieldValue("Lives in Duckburg"));
+        Document inner2 = new Document(manager.getDocumentType("docindoc"), "doc:inner:number:two");
+        inner2.setFieldValue("name", new StringFieldValue("Uncle Scrooge"));
+        inner2.setFieldValue("content", new StringFieldValue("Lives in Duckburg, too."));
+
+        Array<Document> innerArray = (Array<Document>) manager.getDocumentType("outerdoc").getField("innerdocuments").getDataType().createFieldValue();
+        innerArray.add(inner1);
+        innerArray.add(inner2);
+
+        Document outer = new Document(manager.getDocumentType("outerdoc"), "doc:outer:the:only:one");
+        outer.setFieldValue("innerdocuments", innerArray);
+
+        DocumentSerializer serializer = DocumentSerializerFactory.create42();
+        serializer.write(outer);
+
+        GrowableByteBuffer buf = serializer.getBuf();
+        buf.flip();
+
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buf);
+        Document outerDeserialized = new Document(deserializer);
+
+        assertEquals(outer, outerDeserialized);
+        assertNotSame(outer, outerDeserialized);
+
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java b/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java
new file mode 100755
index 00000000000..0e7eb72ee07
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java
@@ -0,0 +1,112 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.ByteFieldValue;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+
+import java.util.HashMap;
+
+/**
+ * @author thomasg
+ */
+public class DocumentCalculatorTestCase extends junit.framework.TestCase {
+
+    DocumentType testDocType = null;
+    DocumentTypeManager docMan;
+    Document doc;
+
+    public void setUp() {
+        docMan = new DocumentTypeManager();
+        testDocType = new DocumentType("testdoc");
+
+        testDocType.addHeaderField("byteattr", DataType.BYTE);
+        testDocType.addHeaderField("intattr", DataType.INT);
+        testDocType.addHeaderField("longattr", DataType.LONG);
+        testDocType.addHeaderField("doubleattr", DataType.DOUBLE);
+        testDocType.addHeaderField("missingattr", DataType.INT);
+
+        docMan.registerDocumentType(testDocType);
+        doc = new Document(testDocType, new DocumentId("doc:testdoc:http://www.ntnu.no/"));
+        doc.setFieldValue(testDocType.getField("byteattr"), new ByteFieldValue((byte)32));
+        doc.setFieldValue(testDocType.getField("intattr"), new IntegerFieldValue(468));
+        doc.setFieldValue(testDocType.getField("longattr"), new LongFieldValue((long)327));
+        doc.setFieldValue(testDocType.getField("doubleattr"), new DoubleFieldValue(25.0));
+    }
+
+    public void testConstant() throws Exception {
+        DocumentCalculator calculator = new DocumentCalculator("4.0");
+        assertEquals(4.0, calculator.evaluate(doc, new HashMap<String, Object>()));
+    }
+
+    public void testSimple() throws Exception {
+        DocumentCalculator calculator = new DocumentCalculator("(3 + 5) / 2");
+        assertEquals(4.0, calculator.evaluate(doc, new HashMap<String, Object>()));
+    }
+
+    public void testDivideByZero() throws Exception {
+        DocumentCalculator calculator = new DocumentCalculator("(3 + 5) / 0");
+        try {
+            System.out.println(calculator.evaluate(doc, new HashMap<String, Object>()));
+            assertTrue(false);
+        } catch (IllegalArgumentException e) {
+            // ok
+        }
+    }
+
+    public void testModByZero() throws Exception {
+        DocumentCalculator calculator = new DocumentCalculator("(3 + 5) % 0");
+        try {
+            System.out.println(calculator.evaluate(doc, new HashMap<String, Object>()));
+            assertTrue(false);
+        } catch (IllegalArgumentException e) {
+            // ok
+        }
+    }
+
+    public void testFieldDivideByZero() throws Exception {
+        try {
+            DocumentCalculator calculator = new DocumentCalculator("(testdoc.byteattr + testdoc.intattr) / testdoc.doubleattr");
+            doc.setFieldValue("doubleattr", new DoubleFieldValue(0.0));
+            calculator.evaluate(doc, new HashMap<String, Object>());
+            assertTrue(false);
+        } catch (IllegalArgumentException e) {
+            // ok
+        }
+    }
+
+    public void testVariables() throws Exception {
+        HashMap<String, Object> vars = new HashMap<String, Object>();
+        vars.put("x", new Double(3.0));
+        vars.put("y", new Double(5.0));
+        DocumentCalculator calculator = new DocumentCalculator("($x + $y) / 2");
+        assertEquals(4.0, calculator.evaluate(doc, vars));
+    }
+
+    public void testFields() throws Exception {
+        DocumentCalculator calculator = new DocumentCalculator("(testdoc.byteattr + testdoc.intattr) / testdoc.doubleattr");
+        assertEquals(20.0, calculator.evaluate(doc, new HashMap<String, Object>()));
+    }
+
+    public void testMissingField() throws Exception {
+        try {
+            DocumentCalculator calculator = new DocumentCalculator("(testdoc.nosuchattribute + testdoc.intattr) / testdoc.doubleattr");
+            calculator.evaluate(doc, new HashMap<String, Object>());
+            assertTrue(false);
+        } catch (IllegalArgumentException e) {
+            // ok
+        }
+    }
+
+    public void testFieldNotSet() throws Exception {
+        try {
+            DocumentCalculator calculator = new DocumentCalculator("(testdoc.missingattr + testdoc.intattr) / testdoc.doubleattr");
+            calculator.evaluate(doc, new HashMap<String, Object>());
+            assertTrue(false);
+        } catch (IllegalArgumentException e) {
+            // ok
+        }
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java
new file mode 100644
index 00000000000..defb86387ab
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java
@@ -0,0 +1,294 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.*;
+import com.yahoo.document.idstring.*;
+
+import java.math.BigInteger;
+
+import java.io.*;
+import java.util.regex.Pattern;
+import java.util.Arrays;
+
+public class DocumentIdTestCase extends junit.framework.TestCase {
+    DocumentTypeManager manager = new DocumentTypeManager();
+
+    public DocumentIdTestCase(String name) {
+        super(name);
+    }
+
+    protected void setUp() {
+        DocumentType testDocType = new DocumentType("testdoc");
+
+        testDocType.addHeaderField("intattr", DataType.INT);
+        testDocType.addField("rawattr", DataType.RAW);
+        testDocType.addField("floatattr", DataType.FLOAT);
+        testDocType.addHeaderField("stringattr", DataType.STRING);
+        testDocType.addHeaderField("Minattr", DataType.INT);
+
+        manager.registerDocumentType(testDocType);
+    }
+
+    public void testCompareTo() {
+        DocumentId docId1 = new Document(manager.getDocumentType("testdoc"), new DocumentId("doc:testdoc:http://www.uio.no/")).getId();
+        DocumentId docId2 = new Document(manager.getDocumentType("testdoc"), new DocumentId("doc:testdoc:http://www.uio.no/")).getId();
+        DocumentId docId3 = new Document(manager.getDocumentType("testdoc"), new DocumentId("doc:testdoc:http://www.ntnu.no/")).getId();
+
+        assertTrue(docId1.equals(docId2));
+        assertTrue(!docId1.equals(docId3));
+        assertTrue(docId1.compareTo(docId3) > 0);
+        assertTrue(docId3.compareTo(docId1) < 0);
+
+        assertEquals(docId1.hashCode(), docId2.hashCode());
+
+    }
+
+    private void checkInvalidUri(String uri) {
+        try {
+            //invalid URI
+            new DocumentId(uri);
+            fail();
+        } catch (IllegalArgumentException iae) {
+        }
+    }
+
+    public void testValidInvalidUriSchemes() {
+        try {
+            //valid URIs
+            new DocumentId("doc:blabla:something");
+            new DocumentId("doc:doc:doc");
+            new DocumentId("userdoc:bla:2387:");
+            new DocumentId("userdoc:bar:0:");
+            new DocumentId("userdoc:bar:18446744073709551615:");
+            new DocumentId("userdoc:foo:15:bar");
+            new DocumentId("id:namespace:type:n=42:whatever");
+            new DocumentId("id:namespace:type::whatever");
+        } catch (IllegalArgumentException iae) {
+            fail(iae.getMessage());
+        }
+
+        checkInvalidUri("foobar:");
+        checkInvalidUri("ballooo:blabla/something/");
+        checkInvalidUri("doc:");
+        checkInvalidUri("doc::");
+        checkInvalidUri("doc:::");
+        checkInvalidUri("doc::/");
+        checkInvalidUri("doc");
+        checkInvalidUri("userdoc:");
+        checkInvalidUri("userdoc::");
+        checkInvalidUri("userdoc:::");
+        checkInvalidUri("userdoc:::/");
+        checkInvalidUri("userdoc");
+        checkInvalidUri("userdoc:-87987//");
+        checkInvalidUri("userdoc:18446744073709551620/bar/");
+        checkInvalidUri("id:namespace:type");
+        checkInvalidUri("id:namespace:type:key-values");
+        checkInvalidUri("id:namespace:type:n=0,n=1:foo");
+        checkInvalidUri("id:namespace:type:g=foo,g=bar:foo");
+        checkInvalidUri("id:namespace:type:n=0,g=foo:foo");
+    }
+
+
+    //Compares globalId with C++ implementation located in
+    // ~document-HEAD/document/src/tests/cpp-globalidbucketids.txt
+    public void testCalculateGlobalId() throws IOException{
+
+        String file = "src/tests/cpp-globalidbucketids.txt";
+        BufferedReader fr = new BufferedReader(new FileReader(file));
+        String line;
+        String[] split_line;
+        String[] split_gid;
+        byte[] b;
+
+        // reads from file
+        while ((line = fr.readLine()) != null) {
+            split_line = line.split(" - ");
+            DocumentId mydoc = new DocumentId(split_line[0]);
+            b = mydoc.getGlobalId();
+            split_gid = Pattern.compile("\\(|\\)").split(split_line[1]);
+            compareStringByte(split_gid[1],b);
+        }
+        fr.close();
+    }
+
+    private void compareStringByte(String s, byte[] b){
+        /*
+        System.out.println("-- "+s+" --");
+        System.out.print("++ 0x");
+        for (int i=0; i<b.length; ++i) {
+            int nr = b[i] & 0xFF;
+            System.out.print(Integer.toHexString(nr / 16) + Integer.toHexString(nr % 16));
+        }
+        System.out.println(" ++");
+        */
+        s = s.substring(2);
+        assertEquals(s.length()/2, b.length);
+        for(int i=0; i<b.length;i++){
+            String ss = s.substring(2*i,2*i+2);
+            assertEquals(Integer.valueOf(ss, 16).intValue(),(((int)b[i])+256)%256);
+        }	
+    }
+
+    //Compares bucketId with C++ implementation located in
+    // ~document-HEAD/document/src/tests/cpp-globalidbucketids.txt
+    public void testGetBucketId() throws IOException{
+        String file = "src/tests/cpp-globalidbucketids.txt";
+        BufferedReader fr = new BufferedReader(new FileReader(file));
+        String line;
+        String[] split_line;
+        BucketId bid;
+
+        // reads from file
+        while ((line = fr.readLine()) != null) {
+            split_line = line.split(" - ");
+            DocumentId mydoc = new DocumentId(split_line[0]);
+            BucketIdFactory factory = new BucketIdFactory(32, 26, 6);
+            bid = new BucketId(factory.getBucketId(mydoc).getId());
+            assertEquals(split_line[2], bid.toString());
+        }
+        fr.close();
+    }
+
+    public void testGroupdoc() {
+        try {
+            //valid
+            new DocumentId("groupdoc:blabla:something:jkl");
+            new DocumentId("groupdoc:doc:doc:asd");
+            new DocumentId("groupdoc:bar:0:a");
+            new DocumentId("groupdoc:bar:18446744073709551615:");
+            new DocumentId("groupdoc:foo:15:bar");
+        } catch (IllegalArgumentException iae) {
+            fail(iae.getMessage());
+        }
+    }
+
+    public void testInvalidGroupdoc() {
+        checkInvalidUri("grouppdoc:blabla:something");
+        checkInvalidUri("groupdoc:blablasomething");
+    }
+
+    public void testUriNamespace() {
+        DocumentId docId = new DocumentId("doc:bar:foo");
+        assertEquals("doc:bar:foo", docId.toString());
+        assertEquals("doc", docId.getScheme().getType().toString());
+        assertEquals("bar", docId.getScheme().getNamespace());
+        assertEquals("foo", docId.getScheme().getNamespaceSpecific());
+
+        docId = new DocumentId("userdoc:ns:90:boo");
+        assertEquals("userdoc:ns:90:boo", docId.toString());
+        assertEquals("userdoc", docId.getScheme().getType().toString());
+        assertEquals("ns", docId.getScheme().getNamespace());
+        assertEquals("boo", docId.getScheme().getNamespaceSpecific());
+        assertEquals(90l, ((UserDocIdString) docId.getScheme()).getUserId());
+
+        docId = new DocumentId("userdoc:ns:18446744073709551615:boo");
+        assertEquals("userdoc:ns:18446744073709551615:boo", docId.toString());
+        assertEquals("userdoc", docId.getScheme().getType().toString());
+        assertEquals("ns", docId.getScheme().getNamespace());
+        assertEquals("boo", docId.getScheme().getNamespaceSpecific());
+        assertEquals(new BigInteger("18446744073709551615").longValue(), ((UserDocIdString) docId.getScheme()).getUserId());
+
+        docId = new DocumentId("userdoc:ns:9223372036854775808:boo");
+        assertEquals("userdoc:ns:9223372036854775808:boo", docId.toString());
+        assertEquals("userdoc", docId.getScheme().getType().toString());
+        assertEquals("ns", docId.getScheme().getNamespace());
+        assertEquals("boo", docId.getScheme().getNamespaceSpecific());
+        assertEquals(new BigInteger("9223372036854775808").longValue(), ((UserDocIdString) docId.getScheme()).getUserId());
+
+        BigInteger negativeUserId = new BigInteger("F00DCAFEDEADBABE", 16);
+        assertEquals(0xF00DCAFEDEADBABEl, negativeUserId.longValue());
+        docId = new DocumentId("userdoc:ns:"+negativeUserId+":bar");
+        assertEquals("userdoc:ns:17297704939806374590:bar", docId.toString());
+        assertEquals(negativeUserId.longValue(), ((UserDocIdString) docId.getScheme()).getUserId());
+
+        docId = new DocumentId("orderdoc(31,19):ns2:1234:1268182861:foo");
+        assertEquals("orderdoc(31,19):ns2:1234:1268182861:foo", docId.toString());
+        assertEquals("orderdoc", docId.getScheme().getType().toString());
+        assertEquals("ns2", docId.getScheme().getNamespace());
+        assertEquals("foo", docId.getScheme().getNamespaceSpecific());
+        assertEquals(31, ((OrderDocIdString)docId.getScheme()).getWidthBits());
+        assertEquals(19, ((OrderDocIdString)docId.getScheme()).getDivisionBits());
+        assertEquals("1234", ((OrderDocIdString)docId.getScheme()).getGroup());
+        assertEquals(1234, ((OrderDocIdString)docId.getScheme()).getUserId());
+        assertEquals(1268182861, ((OrderDocIdString)docId.getScheme()).getOrdering());
+    }
+
+    public void testIdStrings() {
+        DocumentId docId;
+        docId = new DocumentId(new DocIdString("test", "baaaa"));
+        assertEquals("doc:test:baaaa", docId.toString());
+        assertFalse(docId.hasDocType());
+
+        docId = new DocumentId(new UserDocIdString("test", 54, "something"));
+        assertEquals("userdoc:test:54:something", docId.toString());
+        assertFalse(docId.hasDocType());
+
+        docId = new DocumentId(new UserDocIdString("test", 0xFFFFFFFFFFFFFFFFl, "something"));
+        assertEquals("userdoc:test:18446744073709551615:something", docId.toString());
+
+        //sign flipped
+        docId = new DocumentId(new UserDocIdString("test", -8193, "something"));
+        assertEquals("userdoc:test:18446744073709543423:something", docId.toString());
+
+        docId = new DocumentId(new IdIdString("namespace", "type", "g=group", "foobar"));
+        assertEquals("id:namespace:type:g=group:foobar", docId.toString());
+        assertTrue(docId.hasDocType());
+        assertEquals("type", docId.getDocType());
+    }
+
+    public void testIdStringFeatures() {
+        DocumentId none = new DocumentId("id:ns:type::foo");
+        assertFalse(none.getScheme().hasGroup());
+        assertFalse(none.getScheme().hasNumber());
+
+        none = new DocumentId("doc:ns:foo");
+        assertFalse(none.getScheme().hasGroup());
+        assertFalse(none.getScheme().hasNumber());
+
+        DocumentId user = new DocumentId("id:ns:type:n=42:foo");
+        assertFalse(user.getScheme().hasGroup());
+        assertTrue(user.getScheme().hasNumber());
+        assertEquals(42, user.getScheme().getNumber());
+
+        user = new DocumentId("userdoc:ns:42:foo");
+        assertFalse(user.getScheme().hasGroup());
+        assertTrue(user.getScheme().hasNumber());
+        assertEquals(42, user.getScheme().getNumber());
+
+        DocumentId group = new DocumentId("id:ns:type:g=mygroup:foo");
+        assertTrue(group.getScheme().hasGroup());
+        assertFalse(group.getScheme().hasNumber());
+        assertEquals("mygroup", group.getScheme().getGroup());
+
+        group = new DocumentId("groupdoc:ns:mygroup:foo");
+        assertTrue(group.getScheme().hasGroup());
+        assertFalse(group.getScheme().hasNumber());
+        assertEquals("mygroup", group.getScheme().getGroup());
+
+        DocumentId order = new DocumentId("orderdoc(5,2):ns:42:007:foo");
+        assertTrue(order.getScheme().hasGroup());
+        assertTrue(order.getScheme().hasNumber());
+        assertEquals("42", order.getScheme().getGroup());
+        assertEquals(42, order.getScheme().getNumber());
+    }
+
+    public void testHashCodeOfGids() {
+        DocumentId docId0 = new DocumentId("doc:blabla:0");
+        byte[] docId0Gid = docId0.getGlobalId();
+        DocumentId docId0Copy = new DocumentId("doc:blabla:0");
+        byte[] docId0CopyGid = docId0Copy.getGlobalId();
+
+
+        //GIDs should be the same
+        for (int i = 0; i < docId0Gid.length; i++) {
+            assertEquals(docId0Gid[i], docId0CopyGid[i]);
+        }
+
+        //straight hashCode() of byte arrays won't be the same
+        assertFalse(docId0Gid.hashCode() == docId0CopyGid.hashCode());
+
+        //Arrays.hashCode() works better...
+        assertEquals(Arrays.hashCode(docId0Gid), Arrays.hashCode(docId0CopyGid));
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentPathUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentPathUpdateTestCase.java
new file mode 100755
index 00000000000..c414bea2cfa
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentPathUpdateTestCase.java
@@ -0,0 +1,626 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Tests applying and serializing document updates.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DocumentPathUpdateTestCase extends junit.framework.TestCase {
+    DocumentTypeManager docMan;
+
+    DocumentType docType = null;
+    DocumentType docType2 = null;
+
+    public void setUp() {
+        docMan = new DocumentTypeManager();
+
+        docType = new DocumentType("foobar");
+        docType.addField(new Field("num", DataType.INT));
+        docType.addField(new Field("num2", DataType.DOUBLE));
+        docType.addField(new Field("strfoo", DataType.STRING));
+
+        DataType stringarray = DataType.getArray(DataType.STRING);
+        docType.addField(new Field("strarray", stringarray));
+
+        DataType stringwset = DataType.getWeightedSet(DataType.STRING);
+        docType.addField(new Field("strwset", stringwset));
+
+        StructDataType mystructType = new StructDataType("mystruct");
+        mystructType.addField(new Field("title", DataType.STRING));
+        mystructType.addField(new Field("rating", DataType.INT));
+
+        DataType structmap = new MapDataType(DataType.STRING, mystructType);
+        docType.addField(new Field("structmap", structmap));
+
+        DataType structarray = new ArrayDataType(mystructType);
+        docType.addField(new Field("structarray", structarray));
+
+        DataType strmap = new MapDataType(DataType.STRING, DataType.STRING);
+        docType.addField(new Field("strmap", strmap));
+
+        docType.addField(new Field("struct", mystructType));
+
+        docMan.register(docType);
+
+        docType2 = new DocumentType("otherdoctype");
+        docType2.addField(new Field("strinother", DataType.STRING));
+        docMan.register(docType2);
+    }
+
+    public void testRemoveField() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strfoo"));
+        doc.setFieldValue("strfoo", "cocacola");
+        assertEquals(new StringFieldValue("cocacola"), doc.getFieldValue("strfoo"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new RemoveFieldPathUpdate(doc.getDataType(), "strfoo", null));
+        docUp.applyTo(doc);
+        assertNull(doc.getFieldValue("strfoo"));
+   }
+
+    public void testApplyRemoveMultiList() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+        Array<StringFieldValue> strArray = new Array<>(doc.getField("strarray").getDataType());
+        strArray.add(new StringFieldValue("crouching tiger, hidden value"));
+        strArray.add(new StringFieldValue("remove val 1"));
+        strArray.add(new StringFieldValue("hello hello"));
+        doc.setFieldValue("strarray", strArray);
+        assertNotNull(doc.getFieldValue("strarray"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new RemoveFieldPathUpdate(doc.getDataType(), "strarray[$x]", "foobar.strarray[$x] == \"remove val 1\""));
+        docUp.applyTo(doc);
+        assertEquals(2, ((List) doc.getFieldValue("strarray")).size());
+        List docList = (List) doc.getFieldValue("strarray");
+        assertEquals(new StringFieldValue("crouching tiger, hidden value"), docList.get(0));
+        assertEquals(new StringFieldValue("hello hello"), docList.get(1));
+    }
+
+    public void testApplyRemoveEntireListField() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+        Array<StringFieldValue> strArray = new Array<>(doc.getField("strarray").getDataType());
+        strArray.add(new StringFieldValue("this list"));
+        strArray.add(new StringFieldValue("should be"));
+        strArray.add(new StringFieldValue("totally removed"));
+        doc.setFieldValue("strarray", strArray);
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:toast:jam"));
+        docUp.addFieldPathUpdate(new RemoveFieldPathUpdate(doc.getDataType(), "strarray", null));
+        docUp.applyTo(doc);
+        assertNull(doc.getFieldValue("strarray"));
+    }
+
+    public void testApplyRemoveMultiWset() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strwset"));
+        WeightedSet<StringFieldValue> strwset = new WeightedSet<>(doc.getDataType().getField("strwset").getDataType());
+        strwset.put(new StringFieldValue("hello hello"), 10);
+        strwset.put(new StringFieldValue("remove val 1"), 20);
+        doc.setFieldValue("strwset", strwset);
+        assertNotNull(doc.getFieldValue("strwset"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new RemoveFieldPathUpdate(doc.getDataType(), "strwset{remove val 1}", ""));
+        docUp.applyTo(doc);
+        assertEquals(1, ((WeightedSet) doc.getFieldValue("strwset")).size());
+        WeightedSet docWset = (WeightedSet) doc.getFieldValue("strwset");
+        assertEquals(new Integer(10), docWset.get(new StringFieldValue("hello hello")));
+    }
+
+    public void testApplyAssignSingle() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strfoo"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "strfoo", "", new StringFieldValue("something")));
+        docUp.applyTo(doc);
+        assertEquals(new StringFieldValue("something"), doc.getFieldValue("strfoo"));
+    }
+
+    public void testApplyAssignMath() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        doc.setFieldValue(doc.getField("num"), new IntegerFieldValue(34));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "num", "", "($value * 2) / $value"));
+        docUp.applyTo(doc);
+        assertEquals(new IntegerFieldValue(2), doc.getFieldValue(doc.getField("num")));
+    }
+
+    public void testDivideByZero() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        doc.setFieldValue(doc.getField("num"), new IntegerFieldValue(10));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "num", "", "100 / ($value - 10)"));
+        docUp.applyTo(doc);
+        assertEquals(new IntegerFieldValue(10), doc.getFieldValue(doc.getField("num")));
+    }
+
+    public void testAssignMathFieldNotSet() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        doc.setFieldValue(doc.getField("num"), new IntegerFieldValue(10));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "num", "", "100 + foobar.num2"));
+        docUp.applyTo(doc);
+        assertEquals(new IntegerFieldValue(10), doc.getFieldValue(doc.getField("num")));
+    }
+
+    public void testAssignMathMissingField() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        doc.setFieldValue(doc.getField("num"), new IntegerFieldValue(10));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "num", "", "100 + foobar.bogus"));
+        docUp.applyTo(doc);
+        assertEquals(new IntegerFieldValue(10), doc.getFieldValue(doc.getField("num")));
+    }
+
+    public void testAssignMathTargetFieldNotSet() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "num", "", "100"));
+        docUp.applyTo(doc);
+        assertEquals(new IntegerFieldValue(100), doc.getFieldValue(doc.getField("num")));
+    }
+
+    public void testAssignMathTargetFieldNotSetWithValue() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "num", "", "$value + 5"));
+        docUp.applyTo(doc);
+        assertEquals(new IntegerFieldValue(5), doc.getFieldValue(doc.getField("num")));
+    }
+
+    public void testApplyAssignMultiList() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+        Array<StringFieldValue> strArray = new Array<StringFieldValue>(doc.getField("strarray").getDataType());
+        strArray.add(new StringFieldValue("hello hello"));
+        strArray.add(new StringFieldValue("blah blah"));
+        doc.setFieldValue("strarray", strArray);
+        assertNotNull(doc.getFieldValue("strarray"));
+        Array<StringFieldValue> array = new Array<>(doc.getField("strarray").getDataType());
+        array.add(new StringFieldValue("assigned val 0"));
+        array.add(new StringFieldValue("assigned val 1"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "strarray", "", array));
+        docUp.applyTo(doc);
+        assertEquals(2, ((List) doc.getFieldValue("strarray")).size());
+        List docList = (List) doc.getFieldValue("strarray");
+        assertEquals(new StringFieldValue("assigned val 0"), docList.get(0));
+        assertEquals(new StringFieldValue("assigned val 1"), docList.get(1));
+    }
+
+    public void testApplyAssignMultiWlist() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strwset"));
+        WeightedSet<StringFieldValue> strwset = new WeightedSet<>(doc.getDataType().getField("strwset").getDataType());
+        strwset.put(new StringFieldValue("hello hello"), 164);
+        strwset.put(new StringFieldValue("blahdi blahdi"), 243);
+        doc.setFieldValue("strwset", strwset);
+        assertNotNull(doc.getFieldValue("strwset"));
+        WeightedSet<StringFieldValue> assignWset = new WeightedSet<>(docType.getField("strwset").getDataType());
+        assignWset.put(new StringFieldValue("assigned val 0"), 5);
+        assignWset.put(new StringFieldValue("assigned val 1"), 10);
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "strwset", "", assignWset));
+        docUp.applyTo(doc);
+        assertEquals(2, ((WeightedSet) doc.getFieldValue("strwset")).size());
+        WeightedSet docWset = (WeightedSet) doc.getFieldValue("strwset");
+        assertEquals(new Integer(5), docWset.get(new StringFieldValue("assigned val 0")));
+        assertEquals(new Integer(10), docWset.get(new StringFieldValue("assigned val 1")));
+    }
+
+    public void testAssignWsetRemoveIfZero() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue(doc.getField("strwset")));
+        WeightedSet<StringFieldValue> strwset = new WeightedSet<>(doc.getDataType().getField("strwset").getDataType());
+        strwset.put(new StringFieldValue("hello hello"), 164);
+        strwset.put(new StringFieldValue("blahdi blahdi"), 243);
+        doc.setFieldValue(doc.getField("strwset"), strwset);
+        assertNotNull(doc.getFieldValue(doc.getField("strwset")));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        AssignFieldPathUpdate upd = new AssignFieldPathUpdate(doc.getDataType(), "strwset{hello hello}", "", "$value - 164");
+        upd.setRemoveIfZero(true);
+        docUp.addFieldPathUpdate(upd);
+        docUp.applyTo(doc);
+        WeightedSet docWset = (WeightedSet) doc.getFieldValue(doc.getField("strwset"));
+        assertEquals(1, docWset.size());
+        assertEquals(new Integer(243), docWset.get(new StringFieldValue("blahdi blahdi")));
+    }
+
+    public void testApplyAddMultiList() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+
+        Array<StringFieldValue> addList = new Array<StringFieldValue>(doc.getField("strarray").getDataType());
+        addList.add(new StringFieldValue("bo"));
+        addList.add(new StringFieldValue("ba"));
+        addList.add(new StringFieldValue("by"));
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AddFieldPathUpdate(doc.getDataType(), "strarray", "", addList));
+        docUp.applyTo(doc);
+        List<StringFieldValue> values = new ArrayList<>();
+        values.add(new StringFieldValue("bo"));
+        values.add(new StringFieldValue("ba"));
+        values.add(new StringFieldValue("by"));
+        assertEquals(values, doc.getFieldValue("strarray"));
+    }
+
+    public void testAddAndAssignList() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+
+        Array strArray = new Array(doc.getField("strarray").getDataType());
+        strArray.add(new StringFieldValue("hello hello"));
+        strArray.add(new StringFieldValue("blah blah"));
+        doc.setFieldValue("strarray", strArray);
+        assertNotNull(doc.getFieldValue("strarray"));
+
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "strarray[1]", "", new StringFieldValue("assigned val 1")));
+
+        Array adds = new Array(doc.getField("strarray").getDataType());
+        adds.add(new StringFieldValue("new value"));
+
+        docUp.addFieldPathUpdate(new AddFieldPathUpdate(doc.getDataType(), "strarray", "", adds));
+
+        docUp.applyTo(doc);
+        List docList = (List) doc.getFieldValue("strarray");
+        assertEquals(3, docList.size());
+        assertEquals(new StringFieldValue("hello hello"), docList.get(0));
+        assertEquals(new StringFieldValue("assigned val 1"), docList.get(1));
+        assertEquals(new StringFieldValue("new value"), docList.get(2));
+    }
+
+    public void testAssignSimpleMapValueWithVariable()
+    {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        MapFieldValue mfv = new MapFieldValue((MapDataType)doc.getField("strmap").getDataType());
+
+        mfv.put(new StringFieldValue("foo"), new StringFieldValue("bar"));
+        mfv.put(new StringFieldValue("baz"), new StringFieldValue("bananas"));
+        doc.setFieldValue("strmap", mfv);
+
+        // Select on map value, not key
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:hargl:bargl"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "strmap{$x}",
+                "foobar.strmap{$x} == \"bar\"", new StringFieldValue("shinyvalue")));
+        docUp.applyTo(doc);
+
+        MapFieldValue valueNow = (MapFieldValue)doc.getFieldValue("strmap");
+        assertEquals(2, valueNow.size());
+        assertEquals(new StringFieldValue("shinyvalue"), valueNow.get(new StringFieldValue("foo")));
+        assertEquals(new StringFieldValue("bananas"), valueNow.get(new StringFieldValue("baz")));
+    }
+
+    public void testKeyParsing() {
+        assertEquals(new FieldPathEntry.KeyParseResult("", 2), FieldPathEntry.parseKey("{}"));
+        assertEquals(new FieldPathEntry.KeyParseResult("abc", 5), FieldPathEntry.parseKey("{abc}"));
+        assertEquals(new FieldPathEntry.KeyParseResult("abc", 8), FieldPathEntry.parseKey("{   abc}"));
+        // TODO: post-skipping of spaces not supported for verbatim keys in C++. support here?
+        //assertEquals(new FieldPathEntry.KeyParseResult("abc", 8), FieldPathEntry.parseKey("{abc   }"));
+        assertEquals(new FieldPathEntry.KeyParseResult("hello", 9), FieldPathEntry.parseKey("{\"hello\"}"));
+        assertEquals(new FieldPathEntry.KeyParseResult("{abc}", 9), FieldPathEntry.parseKey("{\"{abc}\"}"));
+        assertEquals(new FieldPathEntry.KeyParseResult("abc", 5), FieldPathEntry.parseKey("{abc}stuff"));
+        assertEquals(new FieldPathEntry.KeyParseResult("abc", 10), FieldPathEntry.parseKey("{   \"abc\"}"));
+        assertEquals(new FieldPathEntry.KeyParseResult("abc", 10), FieldPathEntry.parseKey("{\"abc\"   }"));
+        assertEquals(new FieldPathEntry.KeyParseResult("abc", 13), FieldPathEntry.parseKey("{   \"abc\"   }"));
+        // Test quote escaping
+        assertEquals(new FieldPathEntry.KeyParseResult("\"doom house\"", 18), FieldPathEntry.parseKey("{\"\\\"doom house\\\"\"}"));
+        assertEquals(new FieldPathEntry.KeyParseResult("\"", 6), FieldPathEntry.parseKey("{\"\\\"\"}"));
+        assertEquals(new FieldPathEntry.KeyParseResult("a\"b\"c", 11), FieldPathEntry.parseKey("{\"a\\\"b\\\"c\"}"));
+        // Test failure conditions
+        try {
+            FieldPathEntry.parseKey("");
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Key '' does not start with '{'", e.getMessage());
+        }
+
+        try {
+            FieldPathEntry.parseKey("{");
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Key '{' is incomplete. No matching '}'", e.getMessage());
+        }
+
+        try {
+            FieldPathEntry.parseKey("{aaa");
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Key '{aaa' is incomplete. No matching '}'", e.getMessage());
+        }
+
+        try {
+            FieldPathEntry.parseKey("{\"things}");
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Escaped key '{\"things}' is incomplete. No matching '\"'", e.getMessage());
+        }
+
+        try {
+            FieldPathEntry.parseKey("{\"things\\}");
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Escaped key '{\"things\\}' has bad quote character escape sequence. Expected '\"'", e.getMessage());
+        }
+    }
+
+    public void testKeyWithEscapedChars()
+    {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        MapFieldValue mfv = new MapFieldValue((MapDataType)doc.getField("strmap").getDataType());
+
+        mfv.put(new StringFieldValue("here is a \"fancy\" :-} map key :-{"), new StringFieldValue("bar"));
+        mfv.put(new StringFieldValue("baz"), new StringFieldValue("bananas"));
+        doc.setFieldValue("strmap", mfv);
+
+        // Select on map value, not key
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:hargl:bargl"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "strmap{\"here is a \\\"fancy\\\" :-} map key :-{\"}",
+                "", new StringFieldValue("shinyvalue")));
+        docUp.applyTo(doc);
+
+        MapFieldValue valueNow = (MapFieldValue)doc.getFieldValue("strmap");
+        assertEquals(2, valueNow.size());
+        assertEquals(new StringFieldValue("shinyvalue"), valueNow.get(new StringFieldValue("here is a \"fancy\" :-} map key :-{")));
+        assertEquals(new StringFieldValue("bananas"), valueNow.get(new StringFieldValue("baz")));
+    }
+
+    public void testAssignMap() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        MapFieldValue mfv = new MapFieldValue((MapDataType)doc.getField("structmap").getDataType());
+        Struct fv1 = new Struct(mfv.getDataType().getValueType());
+        fv1.setFieldValue("title", new StringFieldValue("thomas"));
+        fv1.setFieldValue("rating", new IntegerFieldValue(32));
+
+        mfv.put(new StringFieldValue("foo"), fv1);
+
+        Struct fv2 = new Struct(mfv.getDataType().getValueType());
+        fv2.setFieldValue("title", new StringFieldValue("cyril"));
+        fv2.setFieldValue("rating", new IntegerFieldValue(16));
+
+        mfv.put(new StringFieldValue("bar"), fv2);
+
+        Struct fv3 = new Struct(mfv.getDataType().getValueType());
+        fv3.setFieldValue("title", new StringFieldValue("ulf"));
+        fv3.setFieldValue("rating", new IntegerFieldValue(8));
+
+        mfv.put(new StringFieldValue("zoo"), fv3);
+
+        doc.setFieldValue("structmap", mfv);
+
+        Struct fv4 = new Struct(mfv.getDataType().getValueType());
+        fv4.setFieldValue("title", new StringFieldValue("tor brede"));
+        fv4.setFieldValue("rating", new IntegerFieldValue(48));
+
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "structmap{bar}", "", fv4));
+        docUp.applyTo(doc);
+
+        MapFieldValue valueNow = (MapFieldValue)doc.getFieldValue("structmap");
+        assertEquals(fv1, valueNow.get(new StringFieldValue("foo")));
+        assertEquals(fv4, valueNow.get(new StringFieldValue("bar")));
+        assertEquals(fv3, valueNow.get(new StringFieldValue("zoo")));
+    }
+
+    public void testAssignMapStruct() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        MapFieldValue mfv = new MapFieldValue((MapDataType)doc.getField("structmap").getDataType());
+        Struct fv1 = new Struct(mfv.getDataType().getValueType());
+        fv1.setFieldValue("title", new StringFieldValue("thomas"));
+        fv1.setFieldValue("rating", new IntegerFieldValue(32));
+
+        mfv.put(new StringFieldValue("foo"), fv1);
+
+        Struct fv2 = new Struct(mfv.getDataType().getValueType());
+        fv2.setFieldValue("title", new StringFieldValue("cyril"));
+        fv2.setFieldValue("rating", new IntegerFieldValue(16));
+
+        mfv.put(new StringFieldValue("bar"), fv2);
+
+        Struct fv3 = new Struct(mfv.getDataType().getValueType());
+        fv3.setFieldValue("title", new StringFieldValue("ulf"));
+        fv3.setFieldValue("rating", new IntegerFieldValue(8));
+
+        mfv.put(new StringFieldValue("zoo"), fv3);
+
+        doc.setFieldValue("structmap", mfv);
+
+        Struct fv4 = new Struct(mfv.getDataType().getValueType());
+        fv4.setFieldValue("title", new StringFieldValue("cyril"));
+        fv4.setFieldValue("rating", new IntegerFieldValue(48));
+
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "structmap{bar}.rating", "", new IntegerFieldValue(48)));
+        docUp.applyTo(doc);
+
+        MapFieldValue valueNow = (MapFieldValue)doc.getFieldValue("structmap");
+        assertEquals(fv1, valueNow.get(new StringFieldValue("foo")));
+        assertEquals(fv4, valueNow.get(new StringFieldValue("bar")));
+        assertEquals(fv3, valueNow.get(new StringFieldValue("zoo")));
+    }
+
+    public void testAssignMapStructVariable() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        MapFieldValue mfv = new MapFieldValue((MapDataType)doc.getField("structmap").getDataType());
+        Struct fv1 = new Struct(mfv.getDataType().getValueType());
+        fv1.setFieldValue(fv1.getField("title"), new StringFieldValue("thomas"));
+        fv1.setFieldValue(fv1.getField("rating"), new IntegerFieldValue(32));
+
+        mfv.put(new StringFieldValue("foo"), fv1);
+
+        Struct fv2 = new Struct(mfv.getDataType().getValueType());
+        fv2.setFieldValue(fv2.getField("title"), new StringFieldValue("cyril"));
+        fv2.setFieldValue(fv2.getField("rating"), new IntegerFieldValue(16));
+
+        mfv.put(new StringFieldValue("bar"), fv2);
+
+        Struct fv3 = new Struct(mfv.getDataType().getValueType());
+        fv3.setFieldValue(fv3.getField("title"), new StringFieldValue("ulf"));
+        fv3.setFieldValue(fv3.getField("rating"), new IntegerFieldValue(8));
+
+        mfv.put(new StringFieldValue("zoo"), fv3);
+
+        doc.setFieldValue(doc.getField("structmap"), mfv);
+
+        Struct fv4 = new Struct(mfv.getDataType().getValueType());
+        fv4.setFieldValue(fv4.getField("title"), new StringFieldValue("cyril"));
+        fv4.setFieldValue(fv4.getField("rating"), new IntegerFieldValue(48));
+
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "structmap{$x}.rating", "foobar.structmap{$x}.title == \"cyril\"", new IntegerFieldValue(48)));
+        docUp.applyTo(doc);
+
+        MapFieldValue valueNow = (MapFieldValue)doc.getFieldValue("structmap");
+        assertEquals(fv1, valueNow.get(new StringFieldValue("foo")));
+        assertEquals(fv4, valueNow.get(new StringFieldValue("bar")));
+        assertEquals(fv3, valueNow.get(new StringFieldValue("zoo")));
+    }
+
+    public void testAssignMapNoexist() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        MapFieldValue mfv = new MapFieldValue((MapDataType)doc.getField("structmap").getDataType());
+
+        Struct fv1 = new Struct(mfv.getDataType().getValueType());
+        fv1.setFieldValue("title", new StringFieldValue("thomas"));
+        fv1.setFieldValue("rating", new IntegerFieldValue(32));
+
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        docUp.addFieldPathUpdate(new AssignFieldPathUpdate(doc.getDataType(), "structmap{foo}", "", fv1));
+        docUp.applyTo(doc);
+
+        MapFieldValue valueNow = (MapFieldValue)doc.getFieldValue("structmap");
+        assertEquals(fv1, valueNow.get(new StringFieldValue("foo")));
+    }
+
+    public void testAssignMapNoexistNocreate() throws Exception {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        MapFieldValue mfv = new MapFieldValue((MapDataType)doc.getField("structmap").getDataType());
+
+        Struct fv1 = new Struct(mfv.getDataType().getValueType());
+        fv1.setFieldValue("title", new StringFieldValue("thomas"));
+        fv1.setFieldValue("rating", new IntegerFieldValue(32));
+
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        AssignFieldPathUpdate ass = new AssignFieldPathUpdate(doc.getDataType(), "structmap{foo}", "", fv1);
+        ass.setCreateMissingPath(false);
+        docUp.addFieldPathUpdate(ass);
+        docUp.applyTo(doc);
+
+        MapFieldValue valueNow = (MapFieldValue)doc.getFieldValue("structmap");
+        assertNull(valueNow);
+    }
+
+    public void testAssignSerialization() throws Exception {
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        AssignFieldPathUpdate ass = new AssignFieldPathUpdate(docType, "num", "", "3");
+        ass.setCreateMissingPath(false);
+        docUp.addFieldPathUpdate(ass);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer();
+        docUp.serialize(DocumentSerializerFactory.createHead(buffer));
+        buffer.flip();
+        DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.createHead(docMan, buffer));
+
+        assertEquals(docUp, docUp2);
+    }
+
+    public void testAddSerialization() throws Exception {
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        Array strArray = new Array(docType.getField("strarray").getDataType());
+        strArray.add(new StringFieldValue("hello hello"));
+        strArray.add(new StringFieldValue("blah blah"));
+
+        AddFieldPathUpdate add = new AddFieldPathUpdate(docType, "strarray", "", strArray);
+        docUp.addFieldPathUpdate(add);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer();
+        docUp.serialize(DocumentSerializerFactory.createHead(buffer));
+        buffer.flip();
+        DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.createHead(docMan, buffer));
+
+        assertEquals(docUp, docUp2);
+    }
+
+    public void testRemoveSerialization() throws Exception {
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+        RemoveFieldPathUpdate remove = new RemoveFieldPathUpdate(docType, "num", "foobar.num > 0");
+        docUp.addFieldPathUpdate(remove);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer();
+        docUp.serialize(DocumentSerializerFactory.createHead(buffer));
+        buffer.flip();
+        DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.createHead(docMan, buffer));
+
+        assertEquals(docUp, docUp2);
+    }
+
+    public void testStartsWith() throws Exception {
+        FieldPath fp1 = docType.buildFieldPath("struct");
+        FieldPath fp2 = docType.buildFieldPath("struct.title");
+        assertTrue(fp2.startsWith(fp1));
+        assertTrue(fp2.startsWith(fp2));
+        assertFalse(fp1.startsWith(fp2));
+    }
+
+    private DocumentUpdate createDocumentUpdateForSerialization() {
+        docMan = DocumentTestCase.setUpCppDocType();
+        docType = docMan.getDocumentType("serializetest");
+
+        DocumentUpdate docUp = new DocumentUpdate(docType, new DocumentId("doc:serialization:xlanguage"));
+
+        AssignFieldPathUpdate ass = new AssignFieldPathUpdate(docType, "intfield", "", "3");
+        ass.setCreateMissingPath(false);
+        ass.setRemoveIfZero(true);
+        docUp.addFieldPathUpdate(ass);
+
+        Array fArray = new Array(docType.getField("arrayoffloatfield").getDataType());
+        fArray.add(new FloatFieldValue(12.0f));
+        fArray.add(new FloatFieldValue(5.0f));
+
+        AddFieldPathUpdate add = new AddFieldPathUpdate(docType, "arrayoffloatfield", "", fArray);
+        docUp.addFieldPathUpdate(add);
+
+        RemoveFieldPathUpdate remove = new RemoveFieldPathUpdate(docType, "intfield", "serializetest.intfield > 0");
+        docUp.addFieldPathUpdate(remove);
+
+        return docUp;
+    }
+
+    public void testGenerateSerializedFile() throws IOException {
+        DocumentUpdate docUp = createDocumentUpdateForSerialization();
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer();
+        docUp.serialize(DocumentSerializerFactory.createHead(buffer));
+
+        int size = buffer.position();
+        buffer.position(0);
+
+        FileOutputStream fos = new FileOutputStream("src/tests/data/serialize-fieldpathupdate-java.dat");
+        fos.write(buffer.array(), 0, size);
+        fos.close();
+    }
+
+    public void testReadSerializedFile() throws IOException {
+        docMan = DocumentTestCase.setUpCppDocType();
+        byte[] data = DocumentTestCase.readFile("src/tests/data/serialize-fieldpathupdate-cpp.dat");
+        DocumentDeserializer buf = DocumentDeserializerFactory.createHead(docMan, GrowableByteBuffer.wrap(data));
+
+        DocumentUpdate upd = new DocumentUpdate(buf);
+
+        DocumentUpdate compare = createDocumentUpdateForSerialization();
+
+        assertEquals(compare, upd);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentRemoveTestCase.java b/document/src/test/java/com/yahoo/document/DocumentRemoveTestCase.java
new file mode 100644
index 00000000000..62d65400d7b
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentRemoveTestCase.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class DocumentRemoveTestCase {
+
+    @Test
+    public void requireThatToStringWorks() {
+        DocumentId docId = new DocumentId("doc:this:is:a:test");
+        DocumentRemove r = new DocumentRemove(docId);
+        assertThat(r.toString().contains(docId.toString()), is(true));
+    }
+
+    @Test
+    public void requireThatEqualsAndHashCodeWorks() {
+        DocumentRemove r1 = new DocumentRemove(new DocumentId("doc:this:is:a:test"));
+        DocumentRemove r2 = new DocumentRemove(new DocumentId("doc:this:is:a:test"));
+        DocumentRemove r3 = new DocumentRemove(new DocumentId("doc:this:is:nonequal"));
+
+        assertThat(r1, equalTo(r1));
+        assertThat(r1, equalTo(r2));
+        assertThat(r2, equalTo(r1));
+        assertThat(r1.hashCode(), equalTo(r2.hashCode()));
+
+        assertThat(r1, not(equalTo(r3)));
+        assertThat(r3, not(equalTo(r1)));
+        assertThat(r2, not(equalTo(r3)));
+        assertThat(r3, not(equalTo(r2)));
+        assertThat(r1.hashCode(), not(equalTo(r3.hashCode())));
+
+        assertThat(r1, not(equalTo(new Object())));
+        assertThat(r1.equals("banana"), is(false));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java
new file mode 100644
index 00000000000..d57d29db81b
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java
@@ -0,0 +1,227 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.compress.CompressionType;
+import com.yahoo.document.annotation.AbstractTypesTest;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.ByteFieldValue;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests serialization of all versions.
+ * <p/>
+ * This test tests serialization and deserialization of documents of all
+ * supported types.
+ * <p/>
+ * Serialization is only supported in newest format. Deserialization should work
+ * for all formats supported, but only the part that makes sense in the new
+ * format. Thus, if new format deprecates a datatype, that datatype, when
+ * serializing old versions, must either just be dropped or converted.
+ * <p/>
+ * Thus, we create document type programmatically, because all old versions need
+ * to make sense with current config.
+ * <p/>
+ * When we create a document programmatically. This is serialized into current
+ * version files. When altering the format, after the alteration, copy the
+ * current version files to a specific version file and add those to list of
+ * files this test checks.
+ * <p/>
+ * When adding new fields to the documents, use the version tagged with each
+ * file to ignore these field for old types.
+ *
+ * @author arnej27959
+ */
+public class DocumentSerializationTestCase extends AbstractTypesTest {
+
+    @Test
+    public void testSerializationAllVersions() throws IOException {
+
+        DocumentType docInDocType = new DocumentType("docindoc");
+        docInDocType.addField(new Field("stringindocfield", DataType.STRING, false));
+
+        DocumentType docType = new DocumentType("serializetest");
+        docType.addField(new Field("floatfield", DataType.FLOAT, true));
+        docType.addField(new Field("stringfield", DataType.STRING, true));
+        docType.addField(new Field("longfield", DataType.LONG, true));
+        docType.addField(new Field("urifield", DataType.URI, true));
+        docType.addField(new Field("intfield", DataType.INT, false));
+        docType.addField(new Field("rawfield", DataType.RAW, false));
+        docType.addField(new Field("doublefield", DataType.DOUBLE, false));
+        docType.addField(new Field("bytefield", DataType.BYTE, false));
+        DataType arrayOfFloatDataType = new ArrayDataType(DataType.FLOAT);
+        docType.addField(new Field("arrayoffloatfield", arrayOfFloatDataType, false));
+        DataType arrayOfArrayOfFloatDataType = new ArrayDataType(arrayOfFloatDataType);
+        docType.addField(new Field("arrayofarrayoffloatfield", arrayOfArrayOfFloatDataType, false));
+        docType.addField(new Field("docfield", DataType.DOCUMENT, false));
+        DataType weightedSetDataType = DataType.getWeightedSet(DataType.STRING, false, false);
+        docType.addField(new Field("wsfield", weightedSetDataType, false));
+
+        DocumentTypeManager docMan = new DocumentTypeManager();
+        docMan.register(docInDocType);
+        docMan.register(docType);
+
+        String path = "src/test/serializeddocuments/";
+
+        {
+            Document doc = new Document(docType, "doc:serializetest:http://test.doc.id/");
+            doc.setFieldValue("intfield", 5);
+            doc.setFieldValue("floatfield", -9.23);
+            doc.setFieldValue("stringfield", "This is a string.");
+            doc.setFieldValue("longfield", new LongFieldValue(398420092938472983l));
+            doc.setFieldValue("doublefield", new DoubleFieldValue(98374532.398820));
+            doc.setFieldValue("bytefield", new ByteFieldValue(254));
+            byte[] rawData = "RAW DATA".getBytes();
+            assertEquals(8, rawData.length);
+            doc.setFieldValue(docType.getField("rawfield"),new Raw(ByteBuffer.wrap("RAW DATA".getBytes())));
+            Document docInDoc = new Document(docInDocType, "doc:serializetest:http://doc.in.doc/");
+            docInDoc.setFieldValue("stringindocfield", "Elvis is dead");
+            doc.setFieldValue(docType.getField("docfield"), docInDoc);
+            Array<FloatFieldValue> floatArray = new Array<>(arrayOfFloatDataType);
+            floatArray.add(new FloatFieldValue(1.0f));
+            floatArray.add(new FloatFieldValue(2.0f));
+            doc.setFieldValue("arrayoffloatfield", floatArray);
+            WeightedSet<StringFieldValue> weightedSet = new WeightedSet<>(weightedSetDataType);
+            weightedSet.put(new StringFieldValue("Weighted 0"), 50);
+            weightedSet.put(new StringFieldValue("Weighted 1"), 199);
+            doc.setFieldValue("wsfield", weightedSet);
+
+            CompressionConfig noncomp = new CompressionConfig();
+            CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
+            {
+                doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
+                doc.getDataType().getBodyType().setCompressionConfig(noncomp);
+                FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-uncompressed.dat", false);
+                doc.serialize(fout);
+                fout.close();
+            }
+            {
+                doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
+                doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
+                FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-lz4-9.dat", false);
+                doc.serialize(fout);
+                doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
+                doc.getDataType().getBodyType().setCompressionConfig(noncomp);
+                fout.close();
+            }
+        }
+
+        class TestDoc {
+
+            String testFile;
+            int version;
+
+            TestDoc(String testFile, int version) {
+                this.testFile = testFile;
+                this.version = version;
+            }
+        }
+
+        String cpppath = "src/tests/data/";
+
+        List<TestDoc> tests = new ArrayList<>();
+        tests.add(new TestDoc(path + "document-java-currentversion-uncompressed.dat",
+                              Document.SERIALIZED_VERSION));
+        tests.add(new TestDoc(path + "document-java-currentversion-lz4-9.dat",
+                              Document.SERIALIZED_VERSION));
+        tests.add(new TestDoc(path + "document-java-v8-uncompressed.dat", 8));
+        tests.add(new TestDoc(cpppath + "document-cpp-currentversion-uncompressed.dat", 7));
+        tests.add(new TestDoc(cpppath + "document-cpp-currentversion-lz4-9.dat", 7));
+        tests.add(new TestDoc(cpppath + "document-cpp-v8-uncompressed.dat", 7));
+        tests.add(new TestDoc(cpppath + "document-cpp-v7-uncompressed.dat", 7));
+        tests.add(new TestDoc(cpppath + "serializev6.dat", 6));
+        for (TestDoc test : tests) {
+            File f = new File(test.testFile);
+            FileInputStream fin = new FileInputStream(f);
+            byte[] buffer = new byte[(int)f.length()];
+            int pos = 0;
+            int remaining = buffer.length;
+            while (remaining > 0) {
+                int read = fin.read(buffer, pos, remaining);
+                assertFalse(read == -1);
+                pos += read;
+                remaining -= read;
+            }
+            System.err.println("Checking doc from file " + test.testFile);
+
+            Document doc = new Document(DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(buffer)));
+
+            System.err.println("Id: " + doc.getId());
+
+            assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield"));
+            assertEquals(-9.23, ((FloatFieldValue)doc.getFieldValue("floatfield")).getFloat(), 1E-6);
+            assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
+            assertEquals(new LongFieldValue(398420092938472983l), doc.getFieldValue("longfield"));
+            assertEquals(98374532.398820, ((DoubleFieldValue)doc.getFieldValue("doublefield")).getDouble(), 1E-6);
+            assertEquals(new ByteFieldValue((byte)254),
+                         doc.getFieldValue("bytefield"));
+            ByteBuffer bbuffer = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer();
+            if (!Arrays.equals("RAW DATA".getBytes(), bbuffer.array())) {
+                System.err.println("Expected 'RAW DATA' but got '"
+                                   + new String(bbuffer.array()) + "'.");
+                assertTrue(false);
+            }
+            if (test.version > 6) {
+                Document docInDoc = (Document)doc.getFieldValue("docfield");
+                assertTrue(docInDoc != null);
+                assertEquals(new StringFieldValue("Elvis is dead"),
+                             docInDoc.getFieldValue("stringindocfield"));
+            }
+            Array array = (Array)doc.getFieldValue("arrayoffloatfield");
+            assertTrue(array != null);
+            assertEquals(1.0f, ((FloatFieldValue)array.get(0)).getFloat(), 1E-6);
+            assertEquals(2.0f, ((FloatFieldValue)array.get(1)).getFloat(), 1E-6);
+            WeightedSet wset = (WeightedSet)doc.getFieldValue("wsfield");
+            assertTrue(wset != null);
+            assertEquals(Integer.valueOf(50), wset.get(new StringFieldValue("Weighted 0")));
+            assertEquals(Integer.valueOf(199), wset.get(new StringFieldValue("Weighted 1")));
+        }
+    }
+
+    @Test
+    public void testSerializeDeserializeWithAnnotations() throws IOException {
+        Document doc = new Document(docType, "doc:foo:bar");
+
+        doc.setFieldValue("age", (byte)123);
+        doc.setFieldValue("story", getAnnotatedString());
+        doc.setFieldValue("date", 13829297);
+        doc.setFieldValue("friend", 2384L);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+        serializer.write(doc);
+        buffer.flip();
+
+        FileOutputStream fos = new FileOutputStream("src/tests/data/serializejavawithannotations.dat");
+        fos.write(buffer.array(), 0, buffer.limit());
+        fos.close();
+
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+        Document doc2 = new Document(deserializer);
+
+        assertEquals(doc, doc2);
+        assertNotSame(doc, doc2);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java
new file mode 100644
index 00000000000..9a9e7042745
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java
@@ -0,0 +1,1407 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.compress.CompressionType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.ByteFieldValue;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FieldPathIteratorHandler;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.vespa.objects.BufferSerializer;
+import org.junit.Test;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test for Document and all its features, including (de)serialization.
+ *
+ * @author <a href="thomasg@yahoo-inc.com>Thomas Gundersen</a>
+ * @author bratseth
+ */
+@SuppressWarnings("deprecation")
+public class DocumentTestCase extends DocumentTestCaseBase {
+
+    private static final String SERTEST_DOC_AS_XML_HEAD =
+            "<document documenttype=\"sertest\" documentid=\"doc:sertest:foobar\">\n" +
+            " <mailid>emailfromalicetobob&amp;someone</mailid>\n" +
+            " <date>-2013512400</date>\n" +
+            " <attachmentcount>2</attachmentcount>\n" +
+            " <rawfield binaryencoding=\"base64\">AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiYw==</rawfield>\n";
+
+    private static final String SERTEST_DOC_AS_XML_WEIGHT1 =
+            " <weightedfield>\n" +
+            "  <item weight=\"10\">this is another test, blah blah</item>\n" +
+            "  <item weight=\"5\">this is a test</item>\n" +
+            " </weightedfield>\n";
+
+    private static final String SERTEST_DOC_AS_XML_WEIGHT2 =
+            " <weightedfield>\n" +
+            "  <item weight=\"5\">this is a test</item>\n" +
+            "  <item weight=\"10\">this is another test, blah blah</item>\n" +
+            " </weightedfield>\n";
+
+    private static final String SERTEST_DOC_AS_XML_SUNNYVALE =
+            " <myposfield>N37.374821;W122.057174</myposfield>\n";
+
+    private static final String SERTEST_DOC_AS_XML_FOOT =
+            " <docindoc documenttype=\"docindoc\" documentid=\"doc:sertest:inserted\">\n" +
+            "  <tull>ball</tull>\n" +
+            " </docindoc>\n" +
+            " <mapfield>\n" +
+            "  <item>\n" +
+            "   <key>foo2</key>\n" +
+            "   <value>bar2</value>\n" +
+            "  </item>\n" +
+            "  <item>\n" +
+            "   <key>foo1</key>\n" +
+            "   <value>bar1</value>\n" +
+            "  </item>\n" +
+            " </mapfield>\n" +
+            SERTEST_DOC_AS_XML_SUNNYVALE +
+            "</document>\n";
+
+    static DocumentTypeManager setUpCppDocType() {
+        return setUpDocType("file:src/tests/data/crossplatform-java-cpp-document.cfg");
+    }
+
+    static DocumentTypeManager setUpDocType(String filename) {
+        DocumentTypeManager dcMan = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(dcMan, filename);
+        return dcMan;
+    }
+
+    public void setUpSertestDocType() {
+        docMan = new DocumentTypeManager();
+
+        DocumentType docInDocType = new DocumentType("docindoc");
+        docInDocType.addField(new Field("tull", 2, docMan.getDataType(2), true));
+
+        docMan.registerDocumentType(docInDocType);
+
+        DocumentType sertestDocType = new DocumentType("sertest");
+        sertestDocType.addField(new Field("mailid", 2, docMan.getDataType(2), true));
+        sertestDocType.addField(new Field("date", 3, docMan.getDataType(0), true));
+        sertestDocType.addField(new Field("from", 4, docMan.getDataType(2), true));
+        sertestDocType.addField(new Field("to", 6, docMan.getDataType(2), true));
+        sertestDocType.addField(new Field("subject", 9, docMan.getDataType(2), true));
+        sertestDocType.addField(new Field("body", 10, docMan.getDataType(2), false));
+        sertestDocType.addField(new Field("attachmentcount", 11, docMan.getDataType(0), false));
+        sertestDocType.addField(new Field("attachments", 1081629685, DataType.getArray(docMan.getDataType(2)), false));
+        sertestDocType.addField(new Field("rawfield", 879, DataType.RAW, false));
+        sertestDocType.addField(new Field("weightedfield", 880, DataType.getWeightedSet(DataType.STRING), false));
+        sertestDocType.addField(new Field("weightedfieldCreate", 881, DataType.getWeightedSet(DataType.STRING, true, false), false));
+        sertestDocType.addField(new Field("docindoc", 882, docInDocType, false));
+        sertestDocType.addField(new Field("mapfield", 883, new MapDataType(DataType.STRING, DataType.STRING), false));
+        sertestDocType.addField(new Field("myposfield", 884, PositionDataType.INSTANCE, false));
+
+        docMan.registerDocumentType(sertestDocType);
+    }
+
+    static byte[] readFile(String filename) throws IOException {
+        FileInputStream fis = new FileInputStream(filename);
+        byte[] data = new byte[1000];
+        int tot = fis.read(data);
+        if (tot == -1) {
+            throw new IOException("Could not read from file " + filename);
+        }
+
+        return data;
+    }
+
+    private Document getSertestDocument() {
+        Document doc = new Document(docMan.getDocumentType("sertest"), new DocumentId("doc:sertest:foobar"));
+        doc.setFieldValue("mailid", "emailfromalicetobob");
+        doc.setFieldValue("date", -2013512400); // 03/13/06 11:00:00
+        doc.setFieldValue("attachmentcount", 2);
+
+        byte[] rawBytes = new byte[100];
+        for (int i = 0; i < rawBytes.length; i++) {
+            rawBytes[i] = (byte)i;
+        }
+
+        doc.setFieldValue("rawfield", new Raw(ByteBuffer.wrap(rawBytes)));
+
+        Document docInDoc = new Document(docMan.getDocumentType("docindoc"), new DocumentId("doc:sertest:inserted"));
+        docInDoc.setFieldValue("tull", "ball");
+        doc.setFieldValue("docindoc", docInDoc);
+
+        WeightedSet<StringFieldValue> wset = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+        wset.put(new StringFieldValue("this is a test"), 5);
+        wset.put(new StringFieldValue("this is another test, blah blah"), 10);
+        doc.setFieldValue("weightedfield", wset);
+
+        MapFieldValue<StringFieldValue, StringFieldValue> map = new MapFieldValue<>(new MapDataType(DataType.STRING, DataType.STRING));
+        map.put(new StringFieldValue("foo1"), new StringFieldValue("bar1"));
+        map.put(new StringFieldValue("foo2"), new StringFieldValue("bar2"));
+        doc.setFieldValue("mapfield", map);
+
+        return doc;
+    }
+
+    @Test
+    public void testTypeChecking() {
+        DocumentType type = new DocumentType("test");
+        type.addField(new Field("double", DataType.DOUBLE));
+        type.addField(new Field("float", DataType.FLOAT));
+        type.addField(new Field("int", DataType.INT));
+        type.addField(new Field("long", DataType.LONG));
+        type.addField(new Field("string", DataType.STRING));
+
+        Document doc = new Document(type, "doc:scheme:");
+        FieldValue stringVal = new StringFieldValue("69");
+        FieldValue doubleVal = new DoubleFieldValue(6.9);
+        FieldValue floatVal = new FloatFieldValue(6.9f);
+        FieldValue intVal = new IntegerFieldValue(69);
+        FieldValue longVal = new LongFieldValue(69L);
+
+        doc.setFieldValue("string", stringVal);
+        doc.setFieldValue("string", doubleVal);
+        doc.setFieldValue("string", floatVal);
+        doc.setFieldValue("string", intVal);
+        doc.setFieldValue("string", longVal);
+
+        doc.setFieldValue("double", stringVal);
+        doc.setFieldValue("double", doubleVal);
+        doc.setFieldValue("double", floatVal);
+        doc.setFieldValue("double", intVal);
+        doc.setFieldValue("double", longVal);
+
+        doc.setFieldValue("float", stringVal);
+        doc.setFieldValue("float", doubleVal);
+        doc.setFieldValue("float", floatVal);
+        doc.setFieldValue("float", intVal);
+        doc.setFieldValue("float", longVal);
+
+        doc.setFieldValue("int", stringVal);
+        doc.setFieldValue("int", doubleVal);
+        doc.setFieldValue("int", floatVal);
+        doc.setFieldValue("int", intVal);
+        doc.setFieldValue("int", longVal);
+
+        doc.setFieldValue("long", stringVal);
+        doc.setFieldValue("long", doubleVal);
+        doc.setFieldValue("long", floatVal);
+        doc.setFieldValue("long", intVal);
+        doc.setFieldValue("long", longVal);
+    }
+
+    class VariableIteratorHandler extends FieldPathIteratorHandler {
+
+        public String retVal = "";
+
+        @Override
+        public void onPrimitive(FieldValue fv) {
+
+            for (Map.Entry<String, IndexValue> entry : getVariables().entrySet()) {
+                retVal += entry.getKey() + ": " + entry.getValue() + ",";
+            }
+            retVal += " - " + fv + "\n";
+        }
+    }
+
+    @Test
+    public void testVariables() {
+        ArrayDataType iarr = new ArrayDataType(DataType.INT);
+        ArrayDataType iiarr = new ArrayDataType(iarr);
+        ArrayDataType iiiarr = new ArrayDataType(iiarr);
+
+        DocumentType type = new DocumentType("test");
+        type.addField(new Field("iiiarray", iiiarr));
+
+        Array<Array<Array<IntegerFieldValue>>> iiiaV = new Array<>(iiiarr);
+        for (int i = 1; i < 4; i++) {
+            Array<Array<IntegerFieldValue>> iiaV = new Array<>(iiarr);
+            for (int j = 1; j < 4; j++) {
+                Array<IntegerFieldValue> iaV = new Array<>(iarr);
+                for (int k = 1; k < 4; k++) {
+                    iaV.add(new IntegerFieldValue(i * j * k));
+                }
+                iiaV.add(iaV);
+            }
+            iiiaV.add(iiaV);
+        }
+
+        Document doc = new Document(type, new DocumentId("doc:foo:testdoc"));
+        doc.setFieldValue("iiiarray", iiiaV);
+
+        {
+            VariableIteratorHandler handler = new VariableIteratorHandler();
+            FieldPath path = type.buildFieldPath("iiiarray[$x][$y][$z]");
+            doc.iterateNested(path, 0, handler);
+
+            String fasit =
+                    "x: 0,y: 0,z: 0, - 1\n" +
+                    "x: 0,y: 0,z: 1, - 2\n" +
+                    "x: 0,y: 0,z: 2, - 3\n" +
+                    "x: 0,y: 1,z: 0, - 2\n" +
+                    "x: 0,y: 1,z: 1, - 4\n" +
+                    "x: 0,y: 1,z: 2, - 6\n" +
+                    "x: 0,y: 2,z: 0, - 3\n" +
+                    "x: 0,y: 2,z: 1, - 6\n" +
+                    "x: 0,y: 2,z: 2, - 9\n" +
+                    "x: 1,y: 0,z: 0, - 2\n" +
+                    "x: 1,y: 0,z: 1, - 4\n" +
+                    "x: 1,y: 0,z: 2, - 6\n" +
+                    "x: 1,y: 1,z: 0, - 4\n" +
+                    "x: 1,y: 1,z: 1, - 8\n" +
+                    "x: 1,y: 1,z: 2, - 12\n" +
+                    "x: 1,y: 2,z: 0, - 6\n" +
+                    "x: 1,y: 2,z: 1, - 12\n" +
+                    "x: 1,y: 2,z: 2, - 18\n" +
+                    "x: 2,y: 0,z: 0, - 3\n" +
+                    "x: 2,y: 0,z: 1, - 6\n" +
+                    "x: 2,y: 0,z: 2, - 9\n" +
+                    "x: 2,y: 1,z: 0, - 6\n" +
+                    "x: 2,y: 1,z: 1, - 12\n" +
+                    "x: 2,y: 1,z: 2, - 18\n" +
+                    "x: 2,y: 2,z: 0, - 9\n" +
+                    "x: 2,y: 2,z: 1, - 18\n" +
+                    "x: 2,y: 2,z: 2, - 27\n";
+
+            assertEquals(fasit, handler.retVal);
+        }
+    }
+
+    @Test
+    public void testGetRecursiveValue() {
+        Document doc = new Document(testDocType, new DocumentId("doc:ns:testdoc"));
+        doc.setFieldValue("primitive1", 1);
+
+        Struct l1s1 = new Struct(doc.getField("l1s1").getDataType());
+        l1s1.setFieldValue("primitive1", 2);
+
+        Struct l2s1 = new Struct(doc.getField("struct2").getDataType());
+        l2s1.setFieldValue("primitive1", 3);
+        l2s1.setFieldValue("primitive2", 4);
+
+        Array<IntegerFieldValue> iarr1 = new Array<>(l2s1.getField("iarray").getDataType());
+        iarr1.add(new IntegerFieldValue(11));
+        iarr1.add(new IntegerFieldValue(12));
+        iarr1.add(new IntegerFieldValue(13));
+        l2s1.setFieldValue("iarray", iarr1);
+
+        ArrayDataType dt = (ArrayDataType)l2s1.getField("sarray").getDataType();
+        Array<Struct> sarr1 = new Array<>(dt);
+        {
+            Struct l3s1 = new Struct(dt.getNestedType());
+            l3s1.setFieldValue("primitive1", 1);
+            l3s1.setFieldValue("primitive2", 2);
+            sarr1.add(l3s1);
+        }
+        {
+            Struct l3s1 = new Struct(dt.getNestedType());
+            l3s1.setFieldValue("primitive1", 1);
+            l3s1.setFieldValue("primitive2", 2);
+            sarr1.add(l3s1);
+        }
+        l2s1.setFieldValue("sarray", sarr1);
+
+        MapFieldValue<StringFieldValue, StringFieldValue> smap1 = new MapFieldValue<>((MapDataType)l2s1.getField("smap").getDataType());
+        smap1.put(new StringFieldValue("leonardo"), new StringFieldValue("dicaprio"));
+        smap1.put(new StringFieldValue("ellen"), new StringFieldValue("page"));
+        smap1.put(new StringFieldValue("joseph"), new StringFieldValue("gordon-levitt"));
+        l2s1.setFieldValue("smap", smap1);
+
+        l1s1.setFieldValue("ss", l2s1.clone());
+
+        MapFieldValue<StringFieldValue, Struct> structmap1 = new MapFieldValue<>((MapDataType)l1s1.getField("structmap").getDataType());
+        structmap1.put(new StringFieldValue("test"), l2s1.clone());
+        l1s1.setFieldValue("structmap", structmap1);
+
+        WeightedSet<StringFieldValue> wset1 = new WeightedSet<>(l1s1.getField("wset").getDataType());
+        wset1.add(new StringFieldValue("foo"));
+        wset1.add(new StringFieldValue("bar"));
+        wset1.add(new StringFieldValue("zoo"));
+        l1s1.setFieldValue("wset", wset1);
+
+        Struct l2s2 = new Struct(doc.getField("struct2").getDataType());
+        l2s2.setFieldValue("primitive1", 5);
+        l2s2.setFieldValue("primitive2", 6);
+
+        WeightedSet<Struct> wset2 = new WeightedSet<>(l1s1.getField("structwset").getDataType());
+        wset2.add(l2s1.clone());
+        wset2.add(l2s2.clone());
+        l1s1.setFieldValue("structwset", wset2);
+
+        doc.setFieldValue("l1s1", l1s1.clone());
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1");
+            assertEquals(l1s1, fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.primitive1");
+            assertEquals(new IntegerFieldValue(2), fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss");
+            assertEquals(l2s1, fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray");
+            assertEquals(iarr1, fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray[2]");
+            assertEquals(new IntegerFieldValue(13), fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray[3]");
+            assertNull(fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.sarray[0].primitive1");
+            assertEquals(new IntegerFieldValue(1), fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.smap{joseph}");
+            assertEquals(new StringFieldValue("gordon-levitt"), fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.smap.key");
+            assertEquals(3, ((Array)fv).size());
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.structmap{test}.primitive1");
+            assertEquals(new IntegerFieldValue(3), fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.structmap.value.primitive1");
+            assertEquals(new IntegerFieldValue(3), fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.wset{foo}");
+            assertEquals(new IntegerFieldValue(1), fv);
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.wset.key");
+            assertEquals(3, ((Array)fv).size());
+        }
+
+        {
+            FieldValue fv = doc.getRecursiveValue("l1s1.structwset.key.primitive1");
+            assertEquals(DataType.INT, (((ArrayDataType)fv.getDataType()).getNestedType()));
+            assertEquals(2, ((Array)fv).size());
+        }
+    }
+
+    class ModifyIteratorHandler extends FieldPathIteratorHandler {
+
+        public ModificationStatus doModify(FieldValue fv) {
+            if (fv instanceof StringFieldValue) {
+                fv.assign("newvalue");
+                return ModificationStatus.MODIFIED;
+            }
+
+            return ModificationStatus.NOT_MODIFIED;
+        }
+
+        public boolean onComplex(FieldValue fv) {
+            return false;
+        }
+    }
+
+    class AddIteratorHandler extends FieldPathIteratorHandler {
+
+        @SuppressWarnings("unchecked")
+        public ModificationStatus doModify(FieldValue fv) {
+            if (fv instanceof Array) {
+                ((Array)fv).add(new IntegerFieldValue(32));
+                return ModificationStatus.MODIFIED;
+            }
+
+            return ModificationStatus.NOT_MODIFIED;
+        }
+
+        public boolean onComplex(FieldValue fv) {
+            return false;
+        }
+    }
+
+    class RemoveIteratorHandler extends FieldPathIteratorHandler {
+
+        public ModificationStatus doModify(FieldValue fv) {
+            return ModificationStatus.REMOVED;
+        }
+
+        public boolean onComplex(FieldValue fv) {
+            return false;
+        }
+    }
+
+    @Test
+    public void testModifyDocument() {
+        Document doc = new Document(testDocType, new DocumentId("doc:ns:testdoc"));
+        doc.setFieldValue("primitive1", 1);
+
+        Struct l1s1 = new Struct(doc.getField("l1s1").getDataType());
+        l1s1.setFieldValue("primitive1", 2);
+
+        Struct l2s1 = new Struct(doc.getField("struct2").getDataType());
+        l2s1.setFieldValue("primitive1", 3);
+        l2s1.setFieldValue("primitive2", 4);
+
+        Array<IntegerFieldValue> iarr1 = new Array<>(l2s1.getField("iarray").getDataType());
+        iarr1.add(new IntegerFieldValue(11));
+        iarr1.add(new IntegerFieldValue(12));
+        iarr1.add(new IntegerFieldValue(13));
+        l2s1.setFieldValue("iarray", iarr1);
+
+        ArrayDataType dt = (ArrayDataType)l2s1.getField("sarray").getDataType();
+        Array<Struct> sarr1 = new Array<>(dt);
+        {
+            Struct l3s1 = new Struct(dt.getNestedType());
+            l3s1.setFieldValue("primitive1", 1);
+            l3s1.setFieldValue("primitive2", 2);
+            sarr1.add(l3s1);
+        }
+        {
+            Struct l3s1 = new Struct(dt.getNestedType());
+            l3s1.setFieldValue("primitive1", 1);
+            l3s1.setFieldValue("primitive2", 2);
+            sarr1.add(l3s1);
+        }
+        l2s1.setFieldValue("sarray", sarr1);
+
+        MapFieldValue<StringFieldValue, StringFieldValue> smap1 = new MapFieldValue<>((MapDataType)l2s1.getField("smap").getDataType());
+        smap1.put(new StringFieldValue("leonardo"), new StringFieldValue("dicaprio"));
+        smap1.put(new StringFieldValue("ellen"), new StringFieldValue("page"));
+        smap1.put(new StringFieldValue("joseph"), new StringFieldValue("gordon-levitt"));
+        l2s1.setFieldValue("smap", smap1);
+
+        l1s1.setFieldValue("ss", l2s1.clone());
+
+        MapFieldValue<StringFieldValue, Struct> structmap1 = new MapFieldValue<>((MapDataType)l1s1.getField("structmap").getDataType());
+        structmap1.put(new StringFieldValue("test"), l2s1.clone());
+        l1s1.setFieldValue("structmap", structmap1);
+
+        WeightedSet<StringFieldValue> wset1 = new WeightedSet<>(l1s1.getField("wset").getDataType());
+        wset1.add(new StringFieldValue("foo"));
+        wset1.add(new StringFieldValue("bar"));
+        wset1.add(new StringFieldValue("zoo"));
+        l1s1.setFieldValue("wset", wset1);
+
+        Struct l2s2 = new Struct(doc.getField("struct2").getDataType());
+        l2s2.setFieldValue("primitive1", 5);
+        l2s2.setFieldValue("primitive2", 6);
+
+        WeightedSet<Struct> wset2 = new WeightedSet<>(l1s1.getField("structwset").getDataType());
+        wset2.add(l2s1.clone());
+        wset2.add(l2s2.clone());
+        l1s1.setFieldValue("structwset", wset2);
+
+        doc.setFieldValue("l1s1", l1s1.clone());
+
+        {
+            ModifyIteratorHandler handler = new ModifyIteratorHandler();
+
+            FieldPath path = doc.getDataType().buildFieldPath("l1s1.structmap.value.smap{leonardo}");
+            doc.iterateNested(path, 0, handler);
+
+            FieldValue fv = doc.getRecursiveValue("l1s1.structmap.value.smap{leonardo}");
+            assertEquals(new StringFieldValue("newvalue"), fv);
+        }
+
+        {
+            AddIteratorHandler handler = new AddIteratorHandler();
+            FieldPath path = doc.getDataType().buildFieldPath("l1s1.ss.iarray");
+            doc.iterateNested(path, 0, handler);
+
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray");
+            assertTrue(((Array)fv).contains(new IntegerFieldValue(32)));
+            assertEquals(4, ((Array)fv).size());
+        }
+
+        {
+            RemoveIteratorHandler handler = new RemoveIteratorHandler();
+            FieldPath path = doc.getDataType().buildFieldPath("l1s1.ss.iarray[1]");
+            doc.iterateNested(path, 0, handler);
+
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray");
+
+            assertFalse(((Array)fv).contains(new Integer(12)));
+            assertEquals(3, ((Array)fv).size());
+        }
+
+        {
+            RemoveIteratorHandler handler = new RemoveIteratorHandler();
+            FieldPath path = doc.getDataType().buildFieldPath("l1s1.ss.iarray[$x]");
+            doc.iterateNested(path, 0, handler);
+
+            FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray");
+            assertEquals(0, ((Array)fv).size());
+        }
+
+        {
+            RemoveIteratorHandler handler = new RemoveIteratorHandler();
+            FieldPath path = doc.getDataType().buildFieldPath("l1s1.structmap.value.smap{leonardo}");
+            doc.iterateNested(path, 0, handler);
+
+            FieldValue fv = doc.getRecursiveValue("l1s1.structmap.value.smap");
+            assertFalse(((MapFieldValue)fv).contains(new StringFieldValue("leonardo")));
+        }
+
+        {
+            RemoveIteratorHandler handler = new RemoveIteratorHandler();
+            FieldPath path = doc.getDataType().buildFieldPath("l1s1.wset{foo}");
+            doc.iterateNested(path, 0, handler);
+
+            FieldValue fv = doc.getRecursiveValue("l1s1.wset");
+            assertFalse(((WeightedSet)fv).contains(new StringFieldValue("foo")));
+        }
+    }
+
+    @Test
+    public void testNoType() {
+        try {
+            new Document(null, new DocumentId("doc:null:URI"));
+            fail("Should have gotten an Exception");
+        } catch (NullPointerException | IllegalArgumentException e) {
+            // Success
+        }
+    }
+
+    @Test
+    public void testURI() {
+        String uri = "doc:testdoc:http://www.ntnu.no/";
+
+        DocumentType documentType = docMan.getDocumentType("testdoc");
+        assertNotNull(documentType);
+        Document doc = new Document(docMan.getDocumentType("testdoc"), new DocumentId(uri));
+        assertEquals(doc.getId().toString(), uri);
+    }
+
+    @Test
+    public void testSetGet() {
+        Document doc = new Document(docMan.getDocumentType("testdoc"), new DocumentId("doc:testdoc:test"));
+        Object val = doc.getFieldValue(minField.getName());
+        assertNull(val);
+        doc.setFieldValue(minField.getName(), 500);
+        val = doc.getFieldValue(minField.getName());
+        assertEquals(new IntegerFieldValue(500), val);
+        val = doc.getFieldValue(minField.getName());
+        assertEquals(new IntegerFieldValue(500), val);
+        doc.removeFieldValue(minField);
+        assertNull(doc.getFieldValue(minField.getName()));
+        assertNull(doc.getFieldValue("doesntexist"));
+    }
+
+    @Test
+    public void testGetField() {
+        Document doc = getTestDocument();
+
+        assertNull(doc.getFieldValue("doesntexist"));
+        assertNull(doc.getFieldValue("notintype"));
+    }
+
+    @Test
+    public void testCppDocCompressed() throws IOException {
+        docMan = setUpCppDocType();
+        byte[] data = readFile("src/test/document/serializecpp-lz4-level9.dat");
+        ByteBuffer buf = ByteBuffer.wrap(data);
+
+        Document doc = docMan.createDocument(new GrowableByteBuffer(buf));
+
+        validateCppDoc(doc);
+    }
+
+    @Test
+    public void testCppDoc() throws IOException {
+        docMan = setUpCppDocType();
+        byte[] data = readFile("src/test/document/serializecpp.dat");
+        ByteBuffer buf = ByteBuffer.wrap(data);
+
+        Document doc = docMan.createDocument(new GrowableByteBuffer(buf));
+        validateCppDoc(doc);
+    }
+
+    @Test
+    public void testV6Doc() throws IOException {
+        docMan = setUpCppDocType();
+        byte[] data = readFile("src/tests/data/serializev6.dat");
+        ByteBuffer buf = ByteBuffer.wrap(data);
+
+        Document doc = docMan.createDocument(new GrowableByteBuffer(buf));
+        validateCppDocNotMap(doc);
+    }
+
+    public void validateCppDoc(Document doc) throws IOException {
+        validateCppDocNotMap(doc);
+        MapFieldValue map = (MapFieldValue)doc.getFieldValue("mapfield");
+        assertEquals(map.get(new StringFieldValue("foo1")), new StringFieldValue("bar1"));
+        assertEquals(map.get(new StringFieldValue("foo2")), new StringFieldValue("bar2"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void validateCppDocNotMap(Document doc) throws IOException {
+        // in practice to validate v6 serialization
+        assertEquals("doc:serializetest:http://test.doc.id/", doc.getId().toString());
+        assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield"));
+        assertEquals(new FloatFieldValue((float)-9.23), doc.getFieldValue("floatfield"));
+        assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
+        assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield"));
+        assertEquals(new DoubleFieldValue(98374532.398820d), doc.getFieldValue("doublefield"));
+        assertEquals(new StringFieldValue("http://this.is.a.test/"), doc.getFieldValue("urifield"));
+        //NOTE: The value really is unsigned 254, which becomes signed -2:
+        assertEquals(new ByteFieldValue(-2), doc.getFieldValue("bytefield"));
+        ByteBuffer raw = ByteBuffer.wrap("RAW DATA".getBytes());
+        assertEquals(new Raw(raw), doc.getFieldValue("rawfield"));
+
+        Document docindoc = (Document)doc.getFieldValue("docfield");
+        assertEquals(docMan.getDocumentType("docindoc"), docindoc.getDataType());
+        assertEquals(new DocumentId("doc:docindoc:http://embedded"), docindoc.getId());
+
+        Array<FloatFieldValue> array = (Array<FloatFieldValue>)doc.getFieldValue("arrayoffloatfield");
+        assertEquals(new FloatFieldValue(1.0f), array.get(0));
+        assertEquals(new FloatFieldValue(2.0f), array.get(1));
+
+        WeightedSet<StringFieldValue> wset = (WeightedSet<StringFieldValue>)doc.getFieldValue("wsfield");
+        assertEquals(new Integer(50), wset.get(new StringFieldValue("Weighted 0")));
+        assertEquals(new Integer(199), wset.get(new StringFieldValue("Weighted 1")));
+    }
+
+    @Test
+    public void testCppDocSplit() throws IOException {
+        docMan = setUpCppDocType();
+        byte[] headerData = readFile("src/test/document/serializecppsplit_header.dat");
+        byte[] bodyData = readFile("src/test/document/serializecppsplit_body.dat");
+
+        DocumentDeserializer header = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(headerData),
+                GrowableByteBuffer.wrap(bodyData));
+
+        Document doc = new Document(header);
+
+        assertEquals("doc:serializetest:http://test.doc.id/", doc.getId().toString());
+        assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield"));
+        assertEquals(new FloatFieldValue((float)-9.23), doc.getFieldValue("floatfield"));
+        assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
+        assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield"));
+        assertEquals(new DoubleFieldValue(98374532.398820d), doc.getFieldValue("doublefield"));
+        assertEquals(new StringFieldValue("http://this.is.a.test/"), doc.getFieldValue("urifield"));
+        //NOTE: The value really is unsigned 254, which becomes signed -2:
+        assertEquals(new ByteFieldValue((byte)-2), doc.getFieldValue("bytefield"));
+        ByteBuffer raw = ByteBuffer.wrap("RAW DATA".getBytes());
+        assertEquals(new Raw(raw), doc.getFieldValue("rawfield"));
+
+        Document docindoc = (Document)doc.getFieldValue("docfield");
+        assertEquals(docMan.getDocumentType("docindoc"), docindoc.getDataType());
+        assertEquals(new DocumentId("doc:docindoc:http://embedded"), docindoc.getId());
+
+        WeightedSet wset = (WeightedSet)doc.getFieldValue("wsfield");
+        assertEquals(new Integer(50), wset.get(new StringFieldValue("Weighted 0")));
+        assertEquals(new Integer(199), wset.get(new StringFieldValue("Weighted 1")));
+    }
+
+    @Test
+    public void testCppDocSplitNoBody() throws IOException {
+        docMan = setUpCppDocType();
+        byte[] headerData = readFile("src/test/document/serializecppsplit_header.dat");
+
+        DocumentDeserializer header = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(headerData));
+
+        Document doc = new Document(header);
+
+        assertEquals("doc:serializetest:http://test.doc.id/", doc.getId().toString());
+        assertEquals(new FloatFieldValue((float)-9.23), doc.getFieldValue("floatfield"));
+        assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
+        assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield"));
+        assertEquals(new StringFieldValue("http://this.is.a.test/"), doc.getFieldValue("urifield"));
+    }
+
+    @Test
+    public void testGenerateSerializedFile() throws IOException {
+
+        docMan = setUpCppDocType();
+        Document doc = new Document(docMan.getDocumentType("serializetest"),
+                                    new DocumentId("doc:serializetest:http://test.doc.id/"));
+
+        Document docindoc = new Document(docMan.getDocumentType("docindoc"),
+                                         new DocumentId("doc:serializetest:http://doc.in.doc/"));
+        docindoc.setFieldValue("stringindocfield", "Elvis is dead");
+        doc.setFieldValue("docfield", docindoc);
+
+        Array<FloatFieldValue> l = new Array<>(doc.getField("arrayoffloatfield").getDataType());
+        l.add(new FloatFieldValue((float)1.0));
+        l.add(new FloatFieldValue((float)2.0));
+        doc.setFieldValue("arrayoffloatfield", l);
+
+        WeightedSet<StringFieldValue>
+                wset = new WeightedSet<>(doc.getDataType().getField("wsfield").getDataType());
+        wset.put(new StringFieldValue("Weighted 0"), 50);
+        wset.put(new StringFieldValue("Weighted 1"), 199);
+        doc.setFieldValue("wsfield", wset);
+
+        MapFieldValue<StringFieldValue, StringFieldValue> map =
+                new MapFieldValue<>(
+                        (MapDataType)doc.getDataType().getField("mapfield").getDataType());
+        map.put(new StringFieldValue("foo1"), new StringFieldValue("bar1"));
+        map.put(new StringFieldValue("foo2"), new StringFieldValue("bar2"));
+        doc.setFieldValue("mapfield", map);
+
+        doc.setFieldValue("bytefield", new ByteFieldValue((byte)254));
+        doc.setFieldValue("rawfield", new Raw(ByteBuffer.wrap("RAW DATA".getBytes())));
+        doc.setFieldValue("intfield", new IntegerFieldValue(5));
+        doc.setFieldValue("floatfield", new FloatFieldValue(-9.23f));
+        doc.setFieldValue("stringfield", new StringFieldValue("This is a string."));
+        doc.setFieldValue("longfield", new LongFieldValue(398420092938472983L));
+        doc.setFieldValue("doublefield", new DoubleFieldValue(98374532.398820d));
+        doc.setFieldValue("urifield", new StringFieldValue("http://this.is.a.test/"));
+
+        int size = doc.getSerializedSize();
+        GrowableByteBuffer buf = new GrowableByteBuffer(size, 2.0f);
+
+        doc.serialize(buf);
+        assertEquals(size, buf.position());
+
+        buf.position(0);
+
+        FileOutputStream fos = new FileOutputStream("src/tests/data/serializejava.dat");
+        fos.write(buf.array(), 0, size);
+        fos.close();
+
+        CompressionConfig noncomp = new CompressionConfig();
+        CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
+
+        doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
+        doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
+        buf = new GrowableByteBuffer(size, 2.0f);
+
+        doc.serialize(buf);
+        doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
+        doc.getDataType().getBodyType().setCompressionConfig(noncomp);
+        fos = new FileOutputStream("src/tests/data/serializejava-compressed.dat");
+        fos.write(buf.array(), 0, buf.position());
+        fos.close();
+    }
+
+    @Test
+    public void testSerializeDeserialize() {
+        setUpSertestDocType();
+        Document doc = getSertestDocument();
+
+        GrowableByteBuffer data = new GrowableByteBuffer();
+        doc.serialize(data);
+        int size = doc.getSerializedSize();
+
+        assertEquals(size, data.position());
+
+        data.flip();
+
+        try {
+            FileOutputStream fos = new FileOutputStream("src/test/files/testser.dat");
+            fos.write(data.array(), 0, data.remaining());
+            fos.close();
+        } catch (Exception e) {
+        }
+
+        Document doc2 = docMan.createDocument(data);
+
+        assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
+        assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
+        assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
+        assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
+        assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
+        assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
+        assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
+        assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
+        byte[] docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array();
+        byte[] doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array();
+        assertEquals(docRawBytes.length, doc2RawBytes.length);
+        for (int i = 0; i < docRawBytes.length; i++) {
+            assertEquals(docRawBytes[i], doc2RawBytes[i]);
+        }
+        assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
+        assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
+        // Do the same thing, splitting document in two
+        DocumentSerializer header = DocumentSerializerFactory.create42(new GrowableByteBuffer(), true);
+        DocumentSerializer body = DocumentSerializerFactory.create42(new GrowableByteBuffer());
+        doc.serializeHeader(header);
+        doc.serializeBody(body);
+        header.getBuf().flip();
+        body.getBuf().flip();
+
+        try {
+            FileOutputStream fos = new FileOutputStream("src/test/files/testser-split.header.dat");
+            fos.write(header.getBuf().array(), 0, header.getBuf().remaining());
+            fos.close();
+            fos = new FileOutputStream("src/test/files/testser-split.body.dat");
+            fos.write(body.getBuf().array(), 0, body.getBuf().remaining());
+            fos.close();
+        } catch (Exception e) {
+        }
+
+        DocumentDeserializer deser = DocumentDeserializerFactory.create42(docMan, header.getBuf(), body.getBuf());
+
+        doc2 = new Document(deser);
+
+        assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
+        assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
+        assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
+        assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
+        assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
+        assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
+        assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
+        assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
+        docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array();
+        doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array();
+        assertEquals(docRawBytes.length, doc2RawBytes.length);
+        for (int i = 0; i < docRawBytes.length; i++) {
+            assertEquals(docRawBytes[i], doc2RawBytes[i]);
+        }
+        assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
+        assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
+
+        Document docInDoc = (Document)doc.getFieldValue("docindoc");
+        assert (docInDoc != null);
+        assertEquals(new StringFieldValue("ball"), docInDoc.getFieldValue("tull"));
+    }
+
+    @Test
+    public void testSerializeDeserializeCompressed() {
+        setUpSertestDocType();
+        Document doc = getSertestDocument();
+
+        CompressionConfig noncomp = new CompressionConfig();
+        CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
+
+        doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
+        doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
+
+        GrowableByteBuffer data = new GrowableByteBuffer();
+        doc.serialize(data);
+        int size = doc.getSerializedSize();
+        doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
+        doc.getDataType().getBodyType().setCompressionConfig(noncomp);
+
+        assertEquals(size, data.position());
+
+        data.flip();
+
+        try {
+            FileOutputStream fos = new FileOutputStream("src/test/files/testser.dat");
+            fos.write(data.array(), 0, data.remaining());
+            fos.close();
+        } catch (Exception e) {
+        }
+
+        Document doc2 = docMan.createDocument(data);
+
+        assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
+        assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
+        assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
+        assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
+        assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
+        assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
+        assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
+        assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
+        byte[] docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array();
+        byte[] doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array();
+        assertEquals(docRawBytes.length, doc2RawBytes.length);
+        for (int i = 0; i < docRawBytes.length; i++) {
+            assertEquals(docRawBytes[i], doc2RawBytes[i]);
+        }
+        assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
+        assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
+
+        // Do the same thing, splitting document in two
+        BufferSerializer header = new BufferSerializer(new GrowableByteBuffer());
+        BufferSerializer body = new BufferSerializer(new GrowableByteBuffer());
+        doc.serializeHeader(header);
+        doc.serializeBody(body);
+        header.getBuf().flip();
+        body.getBuf().flip();
+
+        try {
+            FileOutputStream fos = new FileOutputStream("src/test/files/testser-split.header.dat");
+            fos.write(header.getBuf().array(), 0, header.getBuf().remaining());
+            fos.close();
+            fos = new FileOutputStream("src/test/files/testser-split.body.dat");
+            fos.write(body.getBuf().array(), 0, body.getBuf().remaining());
+            fos.close();
+        } catch (Exception e) {
+        }
+
+        DocumentDeserializer deser = DocumentDeserializerFactory.create42(docMan, header.getBuf(), body.getBuf());
+
+        doc2 = new Document(deser);
+
+        assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
+        assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
+        assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
+        assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
+        assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
+        assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
+        assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
+        assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
+        docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array();
+        doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array();
+        assertEquals(docRawBytes.length, doc2RawBytes.length);
+        for (int i = 0; i < docRawBytes.length; i++) {
+            assertEquals(docRawBytes[i], doc2RawBytes[i]);
+        }
+        assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
+        assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
+    }
+
+    @Test
+    public void testDeserialize() {
+        setUpSertestDocType();
+
+        BufferSerializer buf = new BufferSerializer();
+        try {
+            new Document(DocumentDeserializerFactory.create42(docMan, buf.getBuf()));
+            assertTrue(false);
+        } catch (Exception e) {
+            assertTrue(true);
+        }
+
+        buf = BufferSerializer.wrap("Hello world".getBytes());
+        try {
+            new Document(DocumentDeserializerFactory.create42(docMan, buf.getBuf()));
+            assertTrue(false);
+        } catch (Exception e) {
+            assertTrue(true);
+        }
+    }
+
+    @Test
+    public void testInheritance() {
+        // Create types that inherit each other.. And test that it works..
+        DocumentType parentType = new DocumentType("parent");
+        parentType.addField(new Field("parentbodyint", DataType.INT, false));
+        parentType.addField(new Field("parentheaderint", DataType.INT, true));
+        parentType.addField(new Field("overwritten", DataType.INT, true));
+        DocumentType childType = new DocumentType("child");
+        childType.addField(new Field("childbodyint", DataType.INT, false));
+        childType.addField(new Field("childheaderint", DataType.INT, true));
+        childType.addField(new Field("overwritten", DataType.INT, true));
+        childType.inherit(parentType);
+
+        DocumentTypeManager manager = new DocumentTypeManager();
+        manager.register(childType);
+
+        Document child = new Document(childType, new DocumentId("doc:what:test"));
+        child.setFieldValue(childType.getField("parentbodyint"), new IntegerFieldValue(4));
+        child.setFieldValue("parentheaderint", 6);
+        child.setFieldValue("overwritten", 7);
+        child.setFieldValue("childbodyint", 14);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024, 2f);
+        child.serialize(buffer);
+        buffer.flip();
+        Document childCopy = manager.createDocument(buffer);
+
+        // Test various ways of retrieving values
+        assertEquals(new IntegerFieldValue(4), childCopy.getFieldValue(childType.getField("parentbodyint")));
+        assertEquals(new IntegerFieldValue(6), childCopy.getFieldValue("parentheaderint"));
+        assertEquals(new IntegerFieldValue(7), childCopy.getFieldValue("overwritten"));
+
+        assertEquals(child, childCopy);
+    }
+
+    @Test
+    public void testInheritanceTypeMismatch() {
+        DocumentType parentType = new DocumentType("parent");
+        parentType.addField(new Field("parentbodyint", DataType.INT, false));
+        parentType.addField(new Field("parentheaderint", DataType.INT, true));
+        parentType.addField(new Field("overwritten", DataType.STRING, true));
+        DocumentType childType = new DocumentType("child");
+        childType.addField(new Field("childbodyint", DataType.INT, false));
+        childType.addField(new Field("childheaderint", DataType.INT, true));
+        childType.addField(new Field("overwritten", DataType.INT, true));
+        try {
+            childType.inherit(parentType);
+            fail("Inheritance with conflicting types worked.");
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(),
+                         "Inheritance type mismatch: field \"overwritten\" in datatype \"child\" must have same datatype as in parent document type \"parent\"");
+        }
+    }
+
+    @Test
+    public void testFieldValueImplementations() {
+        docMan = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("impl");
+        docType.addField(new Field("something", DataType.getArray(DataType.STRING), false));
+        docMan.register(docType);
+
+        //just checks that isAssignableFrom() in Document.setFieldValue() goes the right way
+
+        Document doc = new Document(docMan.getDocumentType("impl"), new DocumentId("doc:doctest:fooooobardoc"));
+        Array<StringFieldValue> testlist = new Array<>(doc.getField("something").getDataType());
+        doc.setFieldValue("something", testlist);
+    }
+
+    @Test
+    public void testCompressionConfigured() {
+
+        int size_uncompressed;
+        {
+            DocumentTypeManager docMan = new DocumentTypeManager();
+            docMan.configure("file:src/tests/data/cppdocument.cfg");
+
+            Document doc = new Document(docMan.getDocumentType("serializetest"), new DocumentId("doc:test:test"));
+
+            doc.setFieldValue("stringfield",
+                              "compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me ");
+
+            GrowableByteBuffer data = new GrowableByteBuffer();
+            doc.serialize(data);
+            size_uncompressed = data.position();
+        }
+
+        DocumentTypeManager docMan = new DocumentTypeManager();
+        docMan.configure("file:src/tests/data/compressed.cfg");
+
+        Document doc = new Document(docMan.getDocumentType("serializetest"), new DocumentId("doc:test:test"));
+
+        doc.setFieldValue("stringfield",
+                          "compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me ");
+
+        GrowableByteBuffer data = new GrowableByteBuffer();
+        doc.serialize(data);
+        int size_compressed = data.position();
+
+        assertTrue(size_compressed + " < " + size_uncompressed, size_compressed < size_uncompressed);
+    }
+
+    @Test
+    public void testDocumentDataType() {
+        //use documenttypemanagerconfigurer to read config
+        docMan = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(docMan, "file:src/test/document/docindoc.cfg");
+
+        //get a document type
+        docMan.getDocumentType("outerdoc");
+
+        //create document and necessary structures
+        Document outerdoc = new Document(docMan.getDocumentType("outerdoc"), new DocumentId("doc:recursion:outerdoc"));
+        Document innerdoc = new Document(docMan.getDocumentType("innerdoc"), new DocumentId("doc:recursion:innerdoc"));
+
+        innerdoc.setFieldValue("intfield", 55);
+
+        outerdoc.setFieldValue("stringfield", "boooo");
+        outerdoc.setFieldValue("docfield", innerdoc);
+
+        //serialize document
+        int size = outerdoc.getSerializedSize();
+        GrowableByteBuffer buf = new GrowableByteBuffer(size, 2.0f);
+        outerdoc.serialize(buf);
+
+        assertEquals(size, buf.position());
+
+        //deserialize document
+        buf.position(0);
+        Document outerdoc2 = docMan.createDocument(buf);
+
+        //compare values
+        assertEquals(outerdoc, outerdoc2);
+    }
+
+    @Test
+    public void testTimestamp() {
+        Document doc = new Document(docMan.getDocumentType("testdoc"), new DocumentId("doc:testdoc:timetest"));
+        assertNull(doc.getLastModified());
+        doc.setLastModified(4350129845023985L);
+        assertEquals(new Long(4350129845023985L), doc.getLastModified());
+        doc.setLastModified(null);
+        assertNull(doc.getLastModified());
+
+        Long timestamp = System.currentTimeMillis();
+        doc.setLastModified(timestamp);
+        assertEquals(timestamp, doc.getLastModified());
+
+        GrowableByteBuffer buf = new GrowableByteBuffer();
+        doc.getSerializedSize();
+        doc.serialize(buf);
+        buf.position(0);
+
+        Document doc2 = docMan.createDocument(buf);
+        assertNull(doc2.getLastModified());
+    }
+
+    @Test
+    public void testToXml() {
+        setUpSertestDocType();
+        Document doc = getSertestDocument();
+        doc.setFieldValue("mailid", "emailfromalicetobob&someone");
+        // Pizza Hut Sunnyvale: x="-122057174" y="37374821" latlong="N37.374821;W122.057174"
+        doc.setFieldValue("myposfield", PositionDataType.valueOf(-122057174, 37374821));
+        String xml = doc.toXML(" ");
+        System.out.println(xml);
+        assertTrue(xml.contains(SERTEST_DOC_AS_XML_HEAD));
+        assertTrue(xml.contains(SERTEST_DOC_AS_XML_FOOT));
+        assertTrue(xml.contains(SERTEST_DOC_AS_XML_WEIGHT1) || xml.contains(SERTEST_DOC_AS_XML_WEIGHT2));
+        assertTrue(xml.contains(SERTEST_DOC_AS_XML_SUNNYVALE));
+    }
+
+    @Test
+    public void testXmlSerializer() {
+        setUpSertestDocType();
+        Document doc = getSertestDocument();
+        doc.setFieldValue("mailid", "emailfromalicetobob&someone");
+        doc.setFieldValue("myposfield", PositionDataType.valueOf(-122057174, 37374821));
+        String xml = doc.toXML(" ");
+        XmlDocumentWriter w = XmlDocumentWriter.createWriter(" ");
+        w.write(doc);
+        String otherXml = doc.toXML(" ");
+        assertEquals(xml, otherXml);
+    }
+
+    @Test
+    public void testSingleFieldToXml() {
+        Document doc = new Document(docMan.getDocumentType("testdoc"), new DocumentId("doc:testdoc:xmltest"));
+        doc.setFieldValue("stringattr", new StringFieldValue("hello world"));
+        assertEquals("<value>hello world</value>\n", doc.getFieldValue("stringattr").toXml());
+    }
+
+    @Test
+    public void testDelegatedDocumentToXml() {
+        Document doc = new Document(docMan.getDocumentType("testdoc"), new DocumentId("doc:testdoc:xmltest"));
+        doc.setFieldValue("stringattr", new StringFieldValue("hello universe"));
+        // Should just delegate to toXML
+        assertEquals(
+                "<document documenttype=\"testdoc\" documentid=\"doc:testdoc:xmltest\">\n" +
+                "  <stringattr>hello universe</stringattr>\n" +
+                "</document>\n",
+                doc.toXml());
+    }
+
+    @Test
+    public void testEmptyStringsSerialization() {
+        docMan = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("emptystrings");
+        docType.addField(new Field("emptystring", DataType.STRING));
+        docType.addField(new Field("nullstring", DataType.STRING));
+        docType.addField(new Field("spacestring", DataType.STRING));
+        docType.addField(new Field("astring", DataType.STRING));
+        docMan.registerDocumentType(docType);
+
+        GrowableByteBuffer grbuf = new GrowableByteBuffer();
+        {
+            Document doc = new Document(docType, new DocumentId("doc:a:b:emptystrings"));
+
+            doc.setFieldValue("emptystring", "");
+            doc.removeFieldValue("nullstring");
+            doc.setFieldValue("spacestring", " ");
+            doc.setFieldValue("astring", "a");
+            assertEquals(new StringFieldValue(""), doc.getFieldValue("emptystring"));
+            assertNull(doc.getFieldValue("nullstring"));
+            assertEquals(new StringFieldValue(" "), doc.getFieldValue("spacestring"));
+            assertEquals(new StringFieldValue("a"), doc.getFieldValue("astring"));
+
+            doc.getSerializedSize();
+            doc.serialize(grbuf);
+
+            grbuf.flip();
+        }
+        {
+            Document doc2 = docMan.createDocument(grbuf);
+
+            assertEquals(new StringFieldValue(""), doc2.getFieldValue("emptystring"));
+            assertNull(doc2.getFieldValue("nullstring"));
+            assertEquals(new StringFieldValue(" "), doc2.getFieldValue("spacestring"));
+            assertEquals(new StringFieldValue("a"), doc2.getFieldValue("astring"));
+
+        }
+    }
+
+    @Test
+    public void testBug2354045() {
+        DocumentTypeManager docMan = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("bug2354045");
+        docType.addField(new Field("string", DataType.STRING));
+        docMan.register(docType);
+
+        GrowableByteBuffer grbuf = new GrowableByteBuffer();
+
+        Document doc = new Document(docType, new DocumentId("doc:a:b:strings"));
+
+        doc.removeFieldValue("string");
+        assertNull(doc.getFieldValue("string"));
+
+        doc.getSerializedSize();
+        doc.serialize(grbuf);
+
+        grbuf.flip();
+
+        Document doc2 = docMan.createDocument(grbuf);
+        assertNull(doc2.getFieldValue("string"));
+        assertEquals(doc, doc2);
+    }
+
+    @Test
+    public void testUnknownFieldsDeserialization() {
+        DocumentTypeManager docTypeManasjer = new DocumentTypeManager();
+
+        GrowableByteBuffer buf = new GrowableByteBuffer();
+
+        {
+            DocumentType typeWithDinner = new DocumentType("elvis");
+            typeWithDinner.addField("breakfast", DataType.STRING);
+            typeWithDinner.addField("lunch", DataType.INT);
+            typeWithDinner.addField("dinner", DataType.DOUBLE);
+            docTypeManasjer.registerDocumentType(typeWithDinner);
+
+            Document docWithDinner = new Document(typeWithDinner, "doc:elvis:has:left:the:building");
+            docWithDinner.setFieldValue("breakfast", "peanut butter");
+            docWithDinner.setFieldValue("lunch", 14);
+            docWithDinner.setFieldValue("dinner", 5.43d);
+
+            docWithDinner.serialize(buf);
+            buf.flip();
+
+            docTypeManasjer.clear();
+        }
+
+        {
+            DocumentType typeWithoutDinner = new DocumentType("elvis");
+            typeWithoutDinner.addField("breakfast", DataType.STRING);
+            typeWithoutDinner.addField("lunch", DataType.INT);
+            //no dinner
+            docTypeManasjer.registerDocumentType(typeWithoutDinner);
+
+            Document docWithoutDinner = docTypeManasjer.createDocument(buf);
+
+            assertEquals(new StringFieldValue("peanut butter"), docWithoutDinner.getFieldValue("breakfast"));
+            assertEquals(new IntegerFieldValue(14), docWithoutDinner.getFieldValue("lunch"));
+            assertNull(docWithoutDinner.getFieldValue("dinner"));
+        }
+    }
+
+    @Test
+    public void testBug3233988() {
+        DocumentType type = new DocumentType("foo");
+        Field field = new Field("productdesc", DataType.STRING);
+        type.addField(field);
+
+        Document doc;
+
+        doc = new Document(type, "doc:foo:bar:bar");
+        doc.removeFieldValue("productdesc");
+        assertNull(doc.getFieldValue("productdesc"));
+
+        doc = new Document(type, "doc:foo:bar:bar");
+        assertNull(doc.getFieldValue("productdesc"));
+    }
+
+    @Test
+    public void testRequireThatDocumentWithIdSchemaIdChecksType() {
+        DocumentType docType = new DocumentType("mytype");
+        try {
+            new Document(docType, "id:namespace:mytype::foo");
+        } catch (Exception e) {
+            fail();
+        }
+
+        try {
+            new Document(docType, "id:namespace:wrong-type::foo");
+            fail();
+        } catch (IllegalArgumentException e) {
+        }
+    }
+
+    private class MyDocumentReader implements DocumentReader {
+
+        @Override
+        public void read(Document document) {
+        }
+
+        @Override
+        public DocumentId readDocumentId() {
+            return null;
+        }
+
+        @Override
+        public DocumentType readDocumentType() {
+            return null;
+        }
+
+    }
+
+    @Test
+    public void testRequireThatChangingDocumentTypeChecksId() {
+        MyDocumentReader reader = new MyDocumentReader();
+        Document doc = new Document(reader);
+        doc.setId(new DocumentId("id:namespace:mytype::foo"));
+        DocumentType docType = new DocumentType("mytype");
+        try {
+            doc.setDataType(docType);
+        } catch (Exception e) {
+            fail();
+        }
+        doc = new Document(reader);
+        doc.setId(new DocumentId("id:namespace:mytype::foo"));
+        DocumentType wrongType = new DocumentType("wrongtype");
+        try {
+            doc.setDataType(wrongType);
+            fail();
+        } catch (IllegalArgumentException e) {
+        }
+    }
+
+    @Test
+    public void testDocumentComparisonDoesNotCorruptStateBug6394548() {
+        DocumentTypeManager docMan = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("bug2354045");
+        docType.addField(new Field("string", 2, DataType.STRING, true));
+        docType.addField(new Field("int", 1, DataType.INT, true));
+        docType.addField(new Field("float", 0, DataType.FLOAT, true));
+        docMan.register(docType);
+
+        Document doc1 = new Document(docType, new DocumentId("doc:a:b:bug6394548"));
+        doc1.setFieldValue("string", new StringFieldValue("hello world"));
+        doc1.setFieldValue("int", new IntegerFieldValue(1234));
+        doc1.setFieldValue("float", new FloatFieldValue(5.5f));
+        String doc1Before = doc1.toXml();
+
+        Document doc2 = new Document(docType, new DocumentId("doc:a:b:bug6394548"));
+        doc2.setFieldValue("string", new StringFieldValue("aardvark"));
+        doc2.setFieldValue("int", new IntegerFieldValue(90909));
+        doc2.setFieldValue("float", new FloatFieldValue(777.15f));
+        String doc2Before = doc2.toXml();
+
+        doc1.compareTo(doc2);
+
+        String doc1After = doc1.toXml();
+        String doc2After = doc2.toXml();
+
+        assertEquals(doc1Before, doc1After);
+        assertEquals(doc2Before, doc2After);
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java b/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java
new file mode 100644
index 00000000000..11f2103f238
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.Raw;
+
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertNotNull;
+
+public class DocumentTestCaseBase {
+
+    protected Field byteField = null;
+    protected Field intField = null;
+    protected Field rawField = null;
+    protected Field floatField = null;
+    protected Field stringField = null;
+    protected Field minField = null;
+    protected DocumentType testDocType = null;
+    protected DocumentTypeManager docMan;
+
+    public DocumentTestCaseBase() {
+        docMan = new DocumentTypeManager();
+        testDocType = new DocumentType("testdoc");
+
+        testDocType.addHeaderField("byteattr", DataType.BYTE);
+        testDocType.addHeaderField("intattr", DataType.INT);
+        testDocType.addField("rawattr", DataType.RAW);
+        testDocType.addField("floatattr", DataType.FLOAT);
+        testDocType.addHeaderField("stringattr", DataType.STRING);
+        testDocType.addHeaderField("Minattr", DataType.INT);
+        testDocType.addHeaderField("Minattr2", DataType.INT);
+        testDocType.addHeaderField("primitive1", DataType.INT);
+
+        StructDataType sdt = new StructDataType("struct1");
+        sdt.addField(new Field("primitive1", DataType.INT));
+        sdt.addField(new Field("primitive2", DataType.INT));
+
+        StructDataType sdt2 = new StructDataType("struct2");
+        sdt2.addField(new Field("primitive1", DataType.INT));
+        sdt2.addField(new Field("primitive2", DataType.INT));
+        sdt2.addField(new Field("iarray", new ArrayDataType(DataType.INT)));
+        sdt2.addField(new Field("sarray", new ArrayDataType(sdt)));
+        sdt2.addField(new Field("smap", new MapDataType(DataType.STRING, DataType.STRING)));
+
+        testDocType.addField(new Field("struct2", sdt2));
+
+        StructDataType sdt3 = new StructDataType("struct3");
+        sdt3.addField(new Field("primitive1", DataType.INT));
+        sdt3.addField(new Field("ss", sdt2));
+        sdt3.addField(new Field("structmap", new MapDataType(DataType.STRING, sdt2)));
+        sdt3.addField(new Field("wset", new WeightedSetDataType(DataType.STRING, false, false)));
+        sdt3.addField(new Field("structwset", new WeightedSetDataType(sdt2, false, false)));
+
+        testDocType.addField(new Field("l1s1", sdt3));
+
+        byteField = testDocType.getField("byteattr");
+        intField = testDocType.getField("intattr");
+        rawField = testDocType.getField("rawattr");
+        floatField = testDocType.getField("floatattr");
+        stringField = testDocType.getField("stringattr");
+        minField = testDocType.getField("Minattr");
+
+        docMan.registerDocumentType(testDocType);
+    }
+
+    public Document getTestDocument() {
+        Document doc = new Document(docMan.getDocumentType("testdoc"), new DocumentId("doc:testdoc:http://www.ntnu.no/"));
+        doc.setFieldValue(byteField.getName(), (byte) 30);
+        doc.setFieldValue(byteField.getName(), (byte)30);
+        doc.setFieldValue(intField.getName(), 50);
+
+        ByteBuffer buf = ByteBuffer.allocate(7).put("hei der".getBytes());
+        buf.flip();
+
+        doc.setFieldValue(rawField, new Raw(buf));
+        doc.setFieldValue(floatField, new FloatFieldValue((float) 3.56));
+        doc.setFieldValue("stringattr", "tjohei");
+
+        assertNotNull(doc.getId());
+        assertNotNull(doc.getDataType());
+
+        return doc;
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeIdTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeIdTestCase.java
new file mode 100644
index 00000000000..e086aa63d09
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentTypeIdTestCase.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class DocumentTypeIdTestCase {
+
+    @Test
+    public void requireThatToStringWorks() {
+        DocumentTypeId r = new DocumentTypeId(123);
+        assertThat(r.toString().contains("123"), is(true));
+    }
+
+    @Test
+    public void requireThatEqualsAndHashCodeWorks() {
+        DocumentTypeId r1 = new DocumentTypeId(123);
+        DocumentTypeId r2 = new DocumentTypeId(123);
+        DocumentTypeId r3 = new DocumentTypeId(456);
+
+        assertThat(r1, equalTo(r1));
+        assertThat(r1, equalTo(r2));
+        assertThat(r2, equalTo(r1));
+        assertThat(r1.hashCode(), equalTo(r2.hashCode()));
+
+        assertThat(r1, not(equalTo(r3)));
+        assertThat(r3, not(equalTo(r1)));
+        assertThat(r2, not(equalTo(r3)));
+        assertThat(r3, not(equalTo(r2)));
+        assertThat(r1.hashCode(), not(equalTo(r3.hashCode())));
+
+        assertThat(r1, not(equalTo(new Object())));
+        assertThat(r1.equals("foobar"), is(false));
+    }
+
+    @Test
+    public void requireThatAccessorsWork() {
+        DocumentTypeId r1 = new DocumentTypeId(123);
+        assertThat(r1.getId(), equalTo(123));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
new file mode 100644
index 00000000000..000c2adbb2f
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
@@ -0,0 +1,497 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.annotation.*;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+
+import java.util.Iterator;
+
+/**
+ * @author <a href=TODO>Thomas Gundersen</a>
+ */
+public class DocumentTypeManagerTestCase extends junit.framework.TestCase {
+    public DocumentTypeManagerTestCase(String name) {
+        super(name);
+    }
+
+    // Verify that we can register and retrieve fields.
+    public void testRegisterAndGet() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentType newDocType = new DocumentType("testdoc");
+        newDocType.addField("Fjomp", DataType.INT);
+        newDocType.addHeaderField("Fjols", DataType.STRING);
+
+        manager.registerDocumentType(newDocType);
+
+        DocumentType fetchedDocType = manager.getDocumentType(new DataTypeName("testdoc"));
+
+        Field fetched4 = fetchedDocType.getField("Fjomp");
+
+        assertEquals("Fjomp", fetched4.getName());
+        assertEquals(fetched4.getDataType(), DataType.INT);
+        assertEquals(fetched4.isHeader(), false);
+    }
+
+    public void testBasicTypes() {
+        DocumentTypeManager dtm = new DocumentTypeManager();
+        DataType intType = dtm.getDataType("int");
+        assertSame(DataType.INT, intType);
+
+        DataType stringType = dtm.getDataType("string");
+        assertSame(DataType.STRING, stringType);
+
+        DataType longType = dtm.getDataType("long");
+        assertSame(DataType.LONG, longType);
+
+        DataType doubleType = dtm.getDataType("double");
+        assertSame(DataType.DOUBLE, doubleType);
+    }
+
+    public void testRecursiveRegister() {
+        StructDataType struct = new StructDataType("mystruct");
+        DataType wset1 = DataType.getWeightedSet(DataType.getArray(DataType.INT));
+        DataType wset2 = DataType.getWeightedSet(DataType.getArray(DataType.TAG));
+        struct.addField(new Field("foo", wset1, true));
+        struct.addField(new Field("bar", wset2, false));
+        DataType array = DataType.getArray(struct);
+        DocumentType docType = new DocumentType("mydoc");
+        docType.addField("hmm", array);
+        DocumentType docType2 = new DocumentType("myotherdoc");
+        docType2.addField("myint", DataType.INT);
+        docType2.inherit(docType);
+
+        DocumentTypeManager manager = new DocumentTypeManager();
+        manager.register(docType2);
+
+        assertEquals(struct, manager.getDataType(struct.getId()));
+        assertEquals(struct, manager.getDataType("mystruct"));
+        assertEquals(wset1, manager.getDataType(wset1.getId()));
+        assertEquals(wset2, manager.getDataType(wset2.getId()));
+        assertEquals(array, manager.getDataType(array.getId()));
+        assertEquals(docType, manager.getDataType("mydoc"));
+        assertEquals(docType2, manager.getDataType("myotherdoc"));
+        assertEquals(docType, manager.getDocumentType(new DataTypeName("mydoc")));
+        assertEquals(docType2, manager.getDocumentType(new DataTypeName("myotherdoc")));
+    }
+
+    public void testMultipleDocuments() {
+        DocumentType docType1 = new DocumentType("foo0");
+        docType1.addField("bar", DataType.INT);
+
+        DocumentType docType2 = new DocumentType("foo1");
+        docType2.addField("bar", DataType.STRING);
+
+        DocumentType docType3 = new DocumentType("foo2");
+        docType3.addField("bar", DataType.FLOAT);
+
+        DocumentType docType4 = new DocumentType("foo3");
+        docType4.addField("bar", DataType.RAW);
+
+        DocumentTypeManager manager = new DocumentTypeManager();
+        manager.clear();
+        manager.register(docType1);
+        manager.register(docType2);
+        manager.register(docType3);
+        manager.register(docType4);
+
+        assertSame(docType1, manager.getDocumentType(new DataTypeName("foo0")));
+        assertSame(docType2, manager.getDocumentType(new DataTypeName("foo1")));
+        assertSame(docType3, manager.getDocumentType(new DataTypeName("foo2")));
+        assertSame(docType4, manager.getDocumentType(new DataTypeName("foo3")));
+        
+        assertEquals(manager.getDocumentTypes().size(), 5);
+        assertNotNull(manager.getDocumentTypes().get(new DataTypeName("document")));
+        assertEquals(manager.getDocumentTypes().get(new DataTypeName("foo0")), docType1);
+        assertEquals(manager.getDocumentTypes().get(new DataTypeName("foo0")), new DocumentType("foo0"));
+        assertEquals(manager.getDocumentTypes().get(new DataTypeName("foo1")), docType2);
+    }
+
+    public void testReverseMapOrder() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.map.cfg");
+
+        assertNotNull(manager.getDataType(1000));
+        assertNotNull(manager.getDataType(1001));
+    }
+
+    @SuppressWarnings("deprecation")
+    public void testConfigure() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.cfg");
+
+        Iterator typeIt = manager.documentTypeIterator();
+        DocumentType type = null;
+
+        while (typeIt.hasNext()) {
+            type = (DocumentType) typeIt.next();
+            if (type.getName().equals("foobar")) {
+                break;
+            }
+        }
+        assertNotNull(type);
+
+        assertTrue(type.hasField("foobarfield0"));
+        assertTrue(type.hasField("foobarfield1"));
+
+        Field foobarfield0 = type.getField("foobarfield0");
+        assertTrue(!foobarfield0.isHeader());
+        assertTrue(foobarfield0.getDataType().getCode() == 2);
+
+        Field foobarfield1 = type.getField("foobarfield1");
+        assertTrue(foobarfield1.isHeader());
+        assertTrue(foobarfield1.getDataType().getCode() == 4);
+
+
+        typeIt = manager.documentTypeIterator();
+        while (typeIt.hasNext()) {
+            type = (DocumentType) typeIt.next();
+            if (type.getName().equals("banana")) {
+                break;
+            }
+        }
+
+        assertTrue(type.hasField("bananafield0"));
+
+        Field bananafield0 = type.getField("bananafield0");
+        assertTrue(bananafield0.isHeader());
+        assertTrue(bananafield0.getDataType().getCode() == 16);
+
+        //inheritance:
+        Iterator inhIt = type.getInheritedTypes().iterator();
+
+        DocumentType inhType = (DocumentType) inhIt.next();
+        assertTrue(inhType.getName().equals("foobar"));
+
+        assertFalse(inhIt.hasNext());
+
+        typeIt = manager.documentTypeIterator();
+        while (typeIt.hasNext()) {
+            type = (DocumentType) typeIt.next();
+            if (type.getName().equals("customtypes")) {
+                break;
+            }
+        }
+
+        assertTrue(type.hasField("arrayfloat"));
+        assertTrue(type.hasField("arrayarrayfloat"));
+
+        Field arrayfloat = type.getField("arrayfloat");
+        assertTrue(!arrayfloat.isHeader());
+        ArrayDataType dataType = (ArrayDataType) arrayfloat.getDataType();
+        assertTrue(dataType.getCode() == 99);
+        assertTrue(dataType.getValueClass().equals(Array.class));
+        assertTrue(dataType.getNestedType().getCode() == 1);
+        assertTrue(dataType.getNestedType().getValueClass().equals(FloatFieldValue.class));
+
+
+        Field arrayarrayfloat = type.getField("arrayarrayfloat");
+        ArrayDataType subType = (ArrayDataType) arrayarrayfloat.getDataType();
+        assertTrue(!arrayarrayfloat.isHeader());
+        assertTrue(subType.getCode() == 4003);
+        assertTrue(subType.getValueClass().equals(Array.class));
+        assertTrue(subType.getNestedType().getCode() == 99);
+        assertTrue(subType.getNestedType().getValueClass().equals(Array.class));
+        ArrayDataType subSubType = (ArrayDataType) subType.getNestedType();
+        assertTrue(subSubType.getNestedType().getCode() == 1);
+        assertTrue(subSubType.getNestedType().getValueClass().equals(FloatFieldValue.class));
+    }
+
+    public void testConfigureUpdate() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.cfg");
+
+        DocumentType banana = manager.getDocumentType(new DataTypeName("banana"));
+        DocumentType customtypes = manager.getDocumentType(new DataTypeName("customtypes"));
+
+        assertNull(banana.getField("newfield"));
+        assertEquals(new Field("arrayfloat", 9489, new ArrayDataType(DataType.FLOAT, 99), false), customtypes.getField("arrayfloat"));
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.updated.cfg");
+
+        banana = manager.getDocumentType(new DataTypeName("banana"));
+        customtypes = manager.getDocumentType(new DataTypeName("customtypes"));
+
+        assertEquals(new Field("newfield", 12345, DataType.STRING, true), banana.getField("newfield"));
+        assertNull(customtypes.getField("arrayfloat"));
+    }
+
+    public void testConfigureWithAnnotations() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.annotationtypes1.cfg");
+
+        /*
+  annotation banana {
+    field brand type string { }
+  }
+
+  annotation food {
+    field what type annotationreference<banana> { }
+  }
+
+  annotation cyclic {
+    field blah type annotationreference<cyclic> { }
+  }
+
+  annotation a {
+    field foo type annotationreference<b> { }
+  }
+
+  annotation b {
+  }
+         */
+
+        AnnotationTypeRegistry atr = manager.getAnnotationTypeRegistry();
+        assertEquals(AnnotationTypes.ALL_TYPES.size() + 5, atr.getTypes().size());
+
+        AnnotationType banana = atr.getType("banana");
+        assertTrue(banana.getDataType() instanceof StructDataType);
+        assertEquals("annotation.banana", banana.getDataType().getName());
+        assertEquals(1, ((StructDataType) banana.getDataType()).getFields().size());
+        assertEquals(DataType.STRING, ((StructDataType) banana.getDataType()).getField("brand").getDataType());
+
+        AnnotationType food = atr.getType("food");
+        assertTrue(food.getDataType() instanceof StructDataType);
+        assertEquals("annotation.food", food.getDataType().getName());
+        assertEquals(1, ((StructDataType) food.getDataType()).getFields().size());
+        AnnotationReferenceDataType what = (AnnotationReferenceDataType) ((StructDataType) food.getDataType()).getField("what").getDataType();
+        assertSame(banana, what.getAnnotationType());
+
+        AnnotationType cyclic = atr.getType("cyclic");
+        assertTrue(cyclic.getDataType() instanceof StructDataType);
+        assertEquals("annotation.cyclic", cyclic.getDataType().getName());
+        assertEquals(1, ((StructDataType) cyclic.getDataType()).getFields().size());
+        AnnotationReferenceDataType blah = (AnnotationReferenceDataType) ((StructDataType) cyclic.getDataType()).getField("blah").getDataType();
+        assertSame(cyclic, blah.getAnnotationType());
+
+        AnnotationType b = atr.getType("b");
+        assertNull(b.getDataType());
+
+        AnnotationType a = atr.getType("a");
+        assertTrue(a.getDataType() instanceof StructDataType);
+        assertEquals("annotation.a", a.getDataType().getName());
+        assertEquals(1, ((StructDataType) a.getDataType()).getFields().size());
+        AnnotationReferenceDataType foo = (AnnotationReferenceDataType) ((StructDataType) a.getDataType()).getField("foo").getDataType();
+        assertSame(b, foo.getAnnotationType());
+
+    }
+
+    public void testConfigureWithAnnotationsWithInheritance() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.annotationtypes2.cfg");
+
+
+        /*
+  annotation fruit {
+  }
+
+  annotation banana inherits fruit {
+    field brand type string { }
+  }
+
+  annotation vehicle {
+    field numwheels type int { }
+  }
+
+  annotation car inherits vehicle {
+    field color type string { }
+  }
+
+  annotation intern inherits employee {
+    field enddate type long { }
+  }
+
+  annotation employee inherits worker {
+    field employeeid type int { }
+  }
+
+  annotation worker inherits person {
+  }
+
+  annotation person {
+    field name type string { }
+  }
+         */
+
+        AnnotationTypeRegistry atr = manager.getAnnotationTypeRegistry();
+        assertEquals(AnnotationTypes.ALL_TYPES.size() + 8, atr.getTypes().size());
+
+        AnnotationType fruit = atr.getType("fruit");
+        assertNull(fruit.getDataType());
+
+        AnnotationType banana = atr.getType("banana");
+        assertTrue(banana.getDataType() instanceof StructDataType);
+        assertEquals("annotation.banana", banana.getDataType().getName());
+        assertEquals(1, ((StructDataType) banana.getDataType()).getFields().size());
+        assertEquals(DataType.STRING, ((StructDataType) banana.getDataType()).getField("brand").getDataType());
+        assertEquals(0, ((StructDataType) banana.getDataType()).getInheritedTypes().size());
+
+        AnnotationType vehicle = atr.getType("vehicle");
+        assertTrue(vehicle.getDataType() instanceof StructDataType);
+        assertEquals("annotation.vehicle", vehicle.getDataType().getName());
+        assertEquals(1, ((StructDataType) vehicle.getDataType()).getFields().size());
+        assertEquals(DataType.INT, ((StructDataType) vehicle.getDataType()).getField("numwheels").getDataType());
+        assertEquals(0, ((StructDataType) vehicle.getDataType()).getInheritedTypes().size());
+
+        AnnotationType car = atr.getType("car");
+        assertTrue(car.getDataType() instanceof StructDataType);
+        assertEquals("annotation.car", car.getDataType().getName());
+        assertEquals(2, ((StructDataType) car.getDataType()).getFields().size());
+        assertEquals(DataType.INT, ((StructDataType) car.getDataType()).getField("numwheels").getDataType());
+        assertEquals(DataType.STRING, ((StructDataType) car.getDataType()).getField("color").getDataType());
+        assertEquals(1, ((StructDataType) car.getDataType()).getInheritedTypes().size());
+        assertSame(vehicle.getDataType(), ((StructDataType) car.getDataType()).getInheritedTypes().toArray()[0]);
+
+        AnnotationType person = atr.getType("person");
+        assertTrue(person.getDataType() instanceof StructDataType);
+        assertEquals("annotation.person", person.getDataType().getName());
+        assertEquals(1, ((StructDataType) person.getDataType()).getFields().size());
+        assertEquals(DataType.STRING, ((StructDataType) person.getDataType()).getField("name").getDataType());
+        assertEquals(0, ((StructDataType) person.getDataType()).getInheritedTypes().size());
+
+        AnnotationType worker = atr.getType("worker");
+        assertTrue(worker.getDataType() instanceof StructDataType);
+        assertEquals("annotation.worker", worker.getDataType().getName());
+        assertEquals(1, ((StructDataType) worker.getDataType()).getFields().size());
+        assertEquals(DataType.STRING, ((StructDataType) worker.getDataType()).getField("name").getDataType());
+        assertEquals(1, ((StructDataType) worker.getDataType()).getInheritedTypes().size());
+        assertSame(person.getDataType(), ((StructDataType) worker.getDataType()).getInheritedTypes().toArray()[0]);
+
+        AnnotationType employee = atr.getType("employee");
+        assertTrue(employee.getDataType() instanceof StructDataType);
+        assertEquals("annotation.employee", employee.getDataType().getName());
+        assertEquals(2, ((StructDataType) employee.getDataType()).getFields().size());
+        assertEquals(DataType.STRING, ((StructDataType) employee.getDataType()).getField("name").getDataType());
+        assertEquals(DataType.INT, ((StructDataType) employee.getDataType()).getField("employeeid").getDataType());
+        assertEquals(1, ((StructDataType) employee.getDataType()).getInheritedTypes().size());
+        assertSame(worker.getDataType(), ((StructDataType) employee.getDataType()).getInheritedTypes().toArray()[0]);
+
+        AnnotationType intern = atr.getType("intern");
+        assertTrue(intern.getDataType() instanceof StructDataType);
+        assertEquals("annotation.intern", intern.getDataType().getName());
+        assertEquals(3, ((StructDataType) intern.getDataType()).getFields().size());
+        assertEquals(DataType.STRING, ((StructDataType) intern.getDataType()).getField("name").getDataType());
+        assertEquals(DataType.INT, ((StructDataType) intern.getDataType()).getField("employeeid").getDataType());
+        assertEquals(DataType.LONG, ((StructDataType) intern.getDataType()).getField("enddate").getDataType());
+        assertEquals(1, ((StructDataType) intern.getDataType()).getInheritedTypes().size());
+        assertSame(employee.getDataType(), ((StructDataType) intern.getDataType()).getInheritedTypes().toArray()[0]);
+
+    }
+
+    public void testStructsAnyOrder() {
+        /*
+search annotationsimplicitstruct {
+
+  document annotationsimplicitstruct {
+      field structfield type sct {
+      }
+
+      field structarrayfield type array<sct> {
+      }
+  }
+
+  struct sct {
+    field s1 type string {}
+    field s2 type string {}
+    field s3 type sct {}
+    field s4 type foo {}
+  }
+
+  struct foo {
+    field s1 type int {}
+  }
+}
+         */
+
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.structsanyorder.cfg");
+
+        StructDataType foo = (StructDataType) manager.getDataType("foo");
+        assertNotNull(foo);
+        assertEquals(1, foo.getFields().size());
+        Field foos1 = foo.getField("s1");
+        assertSame(DataType.INT, foos1.getDataType());
+
+        StructDataType sct = (StructDataType) manager.getDataType("sct");
+        assertNotNull(sct);
+        assertEquals(4, sct.getFields().size());
+        Field s1 = sct.getField("s1");
+        assertSame(DataType.STRING, s1.getDataType());
+        Field s2 = sct.getField("s2");
+        assertSame(DataType.STRING, s2.getDataType());
+        Field s3 = sct.getField("s3");
+        assertSame(sct, s3.getDataType());
+        Field s4 = sct.getField("s4");
+        assertSame(foo, s4.getDataType());
+    }
+
+    public void testSombrero1() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.sombrero1.cfg");
+
+        {
+            StringFieldValue sfv = new StringFieldValue("ballooooo");
+            String text = sfv.getString();
+            AnnotationType type = manager.getAnnotationTypeRegistry().getType("base");
+            StructuredFieldValue value = (StructuredFieldValue) type.getDataType().createFieldValue();
+            value.setFieldValue("x", 10);
+            SpanNode span = new Span(0, text.length());
+            SpanTree tree = new SpanTree("span", span);
+            tree.annotate(span, new Annotation(type, value));
+            sfv.setSpanTree(tree);
+        }
+
+        {
+            StringFieldValue sfv = new StringFieldValue("ballooooo");
+            String text = sfv.getString();
+            AnnotationType type = manager.getAnnotationTypeRegistry().getType("derived");
+            StructuredFieldValue value = (StructuredFieldValue) type.getDataType().createFieldValue();
+            value.setFieldValue("x", 10);
+            SpanNode span = new Span(0, text.length());
+            SpanTree tree = new SpanTree("span", span);
+            tree.annotate(span, new Annotation(type, value));
+            sfv.setSpanTree(tree);
+        }
+    }
+
+    public void testPolymorphy() {
+        /*
+  annotation super {
+  }
+  annotation sub inherits super {
+  }
+  annotation blah {
+    field a type annotationreference<super> {}
+  }
+         */
+
+
+        DocumentTypeManager manager = new DocumentTypeManager();
+
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.annotationspolymorphy.cfg");
+
+        AnnotationType suuper = manager.getAnnotationTypeRegistry().getType("super");
+        AnnotationType sub = manager.getAnnotationTypeRegistry().getType("sub");
+
+
+        //reference type for super annotation type
+        AnnotationReferenceDataType refType = (AnnotationReferenceDataType) ((StructDataType) manager.getAnnotationTypeRegistry().getType("blah").getDataType()).getField("a").getDataType();
+
+        Annotation superAnnotation = new Annotation(suuper);
+        Annotation subAnnotation = new Annotation(sub);
+
+        new AnnotationReference(refType, superAnnotation);
+        //this would fail without polymorphy support:
+        new AnnotationReference(refType, subAnnotation);
+
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeTestCase.java
new file mode 100644
index 00000000000..7712cc06c41
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentTypeTestCase.java
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import java.util.Iterator;
+
+/**
+ * TODO: Document purpose
+ *
+ * @author <a href="thomasg@yahoo-inc.com>Thomas Gundersen</a>
+ * @author <a href="bratseth@yahoo-inc.com>Jon S Bratseth</a>
+ */
+public class DocumentTypeTestCase extends junit.framework.TestCase {
+
+    public DocumentTypeTestCase(String name) {
+        super(name);
+    }
+
+    // Verify that we can register and retrieve fields.
+    public void testSetGet() {
+        DocumentType docType = new DocumentType("testdoc");
+        docType.addField("Bongle", DataType.STRING);
+        docType.addField("nalle", DataType.INT);
+
+        assertEquals(docType.getField("Bongle").getName(), "Bongle");
+        assertNull(docType.getField("bongle"));
+
+    }
+
+    public void testInheritance() {
+        DocumentTypeManager typeManager = new DocumentTypeManager();
+
+        DocumentType child = new DocumentType("child");
+        //typeManager.register(child);
+
+        /*
+        ListIterator inherited = child.inheritedIterator();
+        assertTrue(inherited.hasNext());
+        assertEquals(new DataTypeName("document", 0), inherited.next());
+        assertTrue(!inherited.hasNext());
+        */
+        Iterator inherited;
+
+        child.addField("childfield", DataType.INT);
+        child.addField("overridden", DataType.STRING);
+
+        DocumentType parent1 = new DocumentType("parent1");
+        parent1.addField("overridden", DataType.STRING);
+        parent1.addField("parent1field", DataType.STRING);
+        child.inherit(parent1);
+
+        DocumentType parent2 = new DocumentType("parent2");
+        parent2.addField("parent2field", DataType.STRING);
+        child.inherit(parent2);
+
+        DocumentType root = new DocumentType("root");
+        root.addField("rootfield", DataType.STRING);
+        parent1.inherit(root);
+        parent2.inherit(root);
+
+        typeManager.register(root);
+        typeManager.register(parent1);
+        typeManager.register(parent2);
+        typeManager.register(child);
+
+        inherited = child.getInheritedTypes().iterator();
+        assertEquals(parent1, inherited.next());
+        assertEquals(parent2, inherited.next());
+        assertTrue(!inherited.hasNext());
+
+        inherited = parent1.getInheritedTypes().iterator();
+        assertEquals(root, inherited.next());
+        assertTrue(!inherited.hasNext());
+
+        inherited = parent2.getInheritedTypes().iterator();
+        assertEquals(root, inherited.next());
+        assertTrue(!inherited.hasNext());
+
+        inherited = root.getInheritedTypes().iterator();
+        assertEquals(DataType.DOCUMENT, inherited.next());
+        assertTrue(!inherited.hasNext());
+
+        Iterator fields = child.fieldSet().iterator();
+        Field field;
+
+        field = (Field) fields.next();
+        assertEquals("rootfield", field.getName());
+
+        field = (Field) fields.next();
+        assertEquals("overridden", field.getName());
+        assertEquals(DataType.STRING, field.getDataType());
+
+        field = (Field) fields.next();
+        assertEquals("parent1field", field.getName());
+
+        field = (Field) fields.next();
+        assertEquals("parent2field", field.getName());
+
+        field = (Field) fields.next();
+        assertEquals("childfield", field.getName());
+
+        assertFalse(fields.hasNext());
+
+        assert(child.getField("rootfield") != null);
+
+        // TODO: Test uninheriting
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
new file mode 100644
index 00000000000..0e7313991f8
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
@@ -0,0 +1,577 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.document.serialization.*;
+import com.yahoo.document.update.AssignValueUpdate;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.ValueUpdate;
+import com.yahoo.io.GrowableByteBuffer;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+
+/**
+ * Tests applying and serializing document updates.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DocumentUpdateTestCase extends junit.framework.TestCase {
+    DocumentTypeManager docMan;
+
+    DocumentType docType = null;
+    DocumentType docType2 = null;
+    DocumentUpdate docUp = null;
+
+    FieldUpdate assignSingle = null;
+    FieldUpdate assignMultiList = null;
+    FieldUpdate assignMultiWset = null;
+
+    FieldUpdate clearSingleField = null;
+    FieldUpdate removeMultiList = null;
+    FieldUpdate removeMultiWset = null;
+
+    FieldUpdate addSingle = null;
+    FieldUpdate addMultiList = null;
+    FieldUpdate addMultiWset = null;
+
+    public void setUp() {
+        docMan = new DocumentTypeManager();
+
+        docType = new DocumentType("foobar");
+        docType.addField(new Field("strfoo", DataType.STRING));
+
+        DataType stringarray = DataType.getArray(DataType.STRING);
+        docType.addField(new Field("strarray", stringarray));
+
+        DataType stringwset = DataType.getWeightedSet(DataType.STRING);
+        docType.addField(new Field("strwset", stringwset));
+        docMan.register(docType);
+
+        docType2 = new DocumentType("otherdoctype");
+        docType2.addField(new Field("strinother", DataType.STRING));
+        docMan.register(docType2);
+
+        docUp = new DocumentUpdate(docType, new DocumentId("doc:foo:bar"));
+
+        assignSingle = FieldUpdate.createAssign(docType.getField("strfoo"), new StringFieldValue("something"));
+
+        Array<StringFieldValue> assignList = new Array<StringFieldValue>(docType.getField("strarray").getDataType());
+        assignList.add(new StringFieldValue("assigned val 0"));
+        assignList.add(new StringFieldValue("assigned val 1"));
+        assignMultiList = FieldUpdate.createAssign(docType.getField("strarray"), assignList);
+
+        WeightedSet<StringFieldValue> assignWset = new WeightedSet<>(docType.getField("strwset").getDataType());
+        assignWset.put(new StringFieldValue("assigned val 0"), 5);
+        assignWset.put(new StringFieldValue("assigned val 1"), 10);
+        assignMultiWset = FieldUpdate.createAssign(docType.getField("strwset"), assignWset);
+
+        clearSingleField = FieldUpdate.createClearField(docType.getField("strfoo"));
+
+        removeMultiList = FieldUpdate.createRemove(docType.getField("strarray"), new StringFieldValue("remove val 1"));
+
+        removeMultiWset = FieldUpdate.createRemove(docType.getField("strwset"), new StringFieldValue("remove val 1"));
+
+        try {
+            addSingle = FieldUpdate.createAdd(docType.getField("strfoo"), new StringFieldValue("add val"));
+            fail("Shouldn't be able to add to a single-value field");
+        } catch (UnsupportedOperationException uoe) {
+            //success
+        }
+
+        List<StringFieldValue> addList = new ArrayList<>();
+        addList.add(new StringFieldValue("bo"));
+        addList.add(new StringFieldValue("ba"));
+        addList.add(new StringFieldValue("by"));
+        addMultiList = FieldUpdate.createAddAll(docType.getField("strarray"), addList);
+
+        WeightedSet<StringFieldValue> addSet = new WeightedSet<StringFieldValue>(docType.getField("strwset").getDataType());
+        addSet.put(new StringFieldValue("banana"), 137);
+        addSet.put(new StringFieldValue("is"), 143);
+        addSet.put(new StringFieldValue("a"), 157);
+        addSet.put(new StringFieldValue("great"), 163);
+        addSet.put(new StringFieldValue("fruit"), 189);
+        addMultiWset = FieldUpdate.createAddAll(docType.getField("strwset"), addSet);
+    }
+
+    public void testApplyRemoveSingle() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strfoo"));
+        doc.setFieldValue("strfoo", new StringFieldValue("cocacola"));
+        assertEquals(new StringFieldValue("cocacola"), doc.getFieldValue("strfoo"));
+        docUp.addFieldUpdate(clearSingleField);
+        docUp.applyTo(doc);
+        assertNull(doc.getFieldValue("strfoo"));
+    }
+
+    public void testApplyRemoveMultiList() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+        Array<StringFieldValue> strArray = new Array<>(DataType.getArray(DataType.STRING));
+        strArray.add(new StringFieldValue("hello hello"));
+        strArray.add(new StringFieldValue("remove val 1"));
+        doc.setFieldValue("strarray", strArray);
+        assertNotNull(doc.getFieldValue("strarray"));
+        docUp.addFieldUpdate(removeMultiList);
+        docUp.applyTo(doc);
+        assertEquals(1, ((List)doc.getFieldValue("strarray")).size());
+        List docList = (List)doc.getFieldValue("strarray");
+        assertEquals(new StringFieldValue("hello hello"), docList.get(0));
+    }
+
+    public void testApplyRemoveMultiWset() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strwset"));
+        WeightedSet<StringFieldValue> strwset = new WeightedSet<>(doc.getDataType().getField("strwset").getDataType());
+        strwset.put(new StringFieldValue("hello hello"), 10);
+        strwset.put(new StringFieldValue("remove val 1"), 20);
+        doc.setFieldValue("strwset", strwset);
+        assertNotNull(doc.getFieldValue("strwset"));
+        docUp.addFieldUpdate(removeMultiWset);
+        docUp.applyTo(doc);
+        assertEquals(1, ((WeightedSet)doc.getFieldValue("strwset")).size());
+        WeightedSet docWset = (WeightedSet)doc.getFieldValue("strwset");
+        assertEquals(new Integer(10), docWset.get(new StringFieldValue("hello hello")));
+    }
+
+    public void testApplyAssignSingle() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strfoo"));
+        docUp.addFieldUpdate(assignSingle);
+        docUp.applyTo(doc);
+        assertEquals(new StringFieldValue("something"), doc.getFieldValue("strfoo"));
+    }
+
+    public void testApplyAssignMultiList() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+        Array<StringFieldValue> strArray = new Array<>(DataType.getArray(DataType.STRING));
+        strArray.add(new StringFieldValue("hello hello"));
+        strArray.add(new StringFieldValue("blah blah"));
+        doc.setFieldValue("strarray", strArray);
+        assertNotNull(doc.getFieldValue("strarray"));
+        docUp.addFieldUpdate(assignMultiList);
+        docUp.applyTo(doc);
+        assertEquals(2, ((List)doc.getFieldValue("strarray")).size());
+        List docList = (List)doc.getFieldValue("strarray");
+        assertEquals(new StringFieldValue("assigned val 0"), docList.get(0));
+        assertEquals(new StringFieldValue("assigned val 1"), docList.get(1));
+    }
+
+    public void testApplyAssignMultiWlist() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strwset"));
+        WeightedSet<StringFieldValue> strwset = new WeightedSet<>(doc.getDataType().getField("strwset").getDataType());
+        strwset.put(new StringFieldValue("hello hello"), 164);
+        strwset.put(new StringFieldValue("blahdi blahdi"), 243);
+        doc.setFieldValue("strwset", strwset);
+        assertNotNull(doc.getFieldValue("strwset"));
+        docUp.addFieldUpdate(assignMultiWset);
+        docUp.applyTo(doc);
+        assertEquals(2, ((WeightedSet)doc.getFieldValue("strwset")).size());
+        WeightedSet docWset = (WeightedSet)doc.getFieldValue("strwset");
+        assertEquals(new Integer(5), docWset.get(new StringFieldValue("assigned val 0")));
+        assertEquals(new Integer(10), docWset.get(new StringFieldValue("assigned val 1")));
+    }
+
+    public void testApplyAddSingle() {
+        //Unneeded, we can't construct an add for single-value
+    }
+
+    public void testApplyAddMultiList() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strarray"));
+
+        docUp.addFieldUpdate(addMultiList);
+        docUp.applyTo(doc);
+        List<StringFieldValue> values = new ArrayList<>();
+        values.add(new StringFieldValue("bo"));
+        values.add(new StringFieldValue("ba"));
+        values.add(new StringFieldValue("by"));
+        assertEquals(values, doc.getFieldValue("strarray"));
+    }
+
+    public void testApplyAddMultiWset() {
+        Document doc = new Document(docMan.getDocumentType("foobar"), new DocumentId("doc:something:foooo"));
+        assertNull(doc.getFieldValue("strwset"));
+
+        WeightedSet<StringFieldValue> wset = new WeightedSet<>(doc.getDataType().getField("strwset").getDataType());
+        wset.put(new StringFieldValue("this"), 5);
+        wset.put(new StringFieldValue("is"), 10);
+        wset.put(new StringFieldValue("a"), 15);
+        wset.put(new StringFieldValue("test"), 20);
+        doc.setFieldValue("strwset", wset);
+
+        docUp.addFieldUpdate(addMultiWset);
+        docUp.applyTo(doc);
+
+        WeightedSet<StringFieldValue> values = new WeightedSet<>(doc.getDataType().getField("strwset").getDataType());
+        values.put(new StringFieldValue("this"), 5);
+        values.put(new StringFieldValue("is"), 143);
+        values.put(new StringFieldValue("a"), 157);
+        values.put(new StringFieldValue("test"), 20);
+        values.put(new StringFieldValue("banana"), 137);
+        values.put(new StringFieldValue("great"), 163);
+        values.put(new StringFieldValue("fruit"), 189);
+        assertEquals(values, doc.getFieldValue("strwset"));
+    }
+
+    public void testUpdatesToTheSameFieldAreCombining() {
+        DocumentType docType = new DocumentType("my_type");
+        Field field = new Field("my_int", DataType.INT);
+        docType.addField(field);
+
+        DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:foo:"));
+        update.addFieldUpdate(FieldUpdate.createAssign(field, new IntegerFieldValue(1)));
+        update.addFieldUpdate(FieldUpdate.createAssign(field, new IntegerFieldValue(2)));
+
+        assertEquals(1, update.getFieldUpdates().size());
+        FieldUpdate fieldUpdate = update.getFieldUpdate(0);
+        assertNotNull(fieldUpdate);
+        assertEquals(field, fieldUpdate.getField());
+        assertEquals(2, fieldUpdate.getValueUpdates().size());
+        ValueUpdate valueUpdate = fieldUpdate.getValueUpdate(0);
+        assertNotNull(valueUpdate);
+        assertTrue(valueUpdate instanceof AssignValueUpdate);
+        assertEquals(new IntegerFieldValue(1), valueUpdate.getValue());
+        assertNotNull(valueUpdate = fieldUpdate.getValueUpdate(1));
+        assertTrue(valueUpdate instanceof AssignValueUpdate);
+        assertEquals(new IntegerFieldValue(2), valueUpdate.getValue());
+    }
+
+    public void testUpdateToWrongField() {
+        DocumentType docType = new DocumentType("my_type");
+        DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:foo:"));
+        try {
+            update.addFieldUpdate(FieldUpdate.createIncrement(new Field("my_int", DataType.INT), 1));
+            fail();
+        } catch (IllegalArgumentException e) {
+            System.out.println(e.getMessage());
+        }
+    }
+
+    public void testSerialize() {
+        docUp.addFieldUpdate(assignSingle);
+        docUp.addFieldUpdate(addMultiList);
+
+        DocumentDeserializer buf = DocumentDeserializerFactory.create42(docMan, new GrowableByteBuffer());
+        docUp.serialize((DocumentUpdateWriter)buf);
+        buf.getBuf().flip();
+
+        try {
+            FileOutputStream fos = new FileOutputStream("src/test/files/updateser.dat");
+            fos.write(buf.getBuf().array(), 0, buf.getBuf().remaining());
+            fos.close();
+        } catch (Exception e) {
+        }
+
+        assertEquals(2 // version
+                     + (11 + 1) //docid doc:foo:bar\0
+                     + 1 //contents
+                     + (6 + 1 + 2) //doctype foobar\0\0\0
+                     + 4 //num field updates
+
+                     //field update 1:
+                     + (4 //field id
+                        + 4 //num valueupdates
+
+                        + (4 //valueUpdateClassID
+                           + 1 //valuepresent
+                           + (1 + 1 + 9 + 1)) //value
+
+                        //field update 2
+                        + (4 //field id
+                           + 4 //num valueupdates
+
+                           + (4  //valueUpdateClassID
+                              + (4 + 4 + 4 + (1 + 1 + 2 + 1) + 4 + (1 + 1 + 2 + 1) + 4 + (1 + 1 + 2 + 1))))) //value
+                , buf.getBuf().remaining());
+
+        DocumentUpdate docUpDeser = new DocumentUpdate(buf);
+
+        assertEquals(docUp.getDocumentType(), docUpDeser.getDocumentType());
+        assertEquals(docUp, docUpDeser);
+    }
+
+    public void testCppDocUpd() throws IOException {
+        docMan = DocumentTestCase.setUpCppDocType();
+        byte[] data = DocumentTestCase.readFile("src/tests/data/serializeupdatecpp.dat");
+        DocumentDeserializer buf = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(data));
+
+        DocumentType type = docMan.getDocumentType("serializetest");
+
+        DocumentUpdate upd = new DocumentUpdate(buf);
+
+        assertEquals(new DocumentId("doc:update:test"), upd.getId());
+        assertEquals(type, upd.getType());
+
+        FieldUpdate serAssignFU = upd.getFieldUpdate(0);
+        assertEquals(type.getField("intfield"), serAssignFU.getField());
+        ValueUpdate serAssign = serAssignFU.getValueUpdate(0);
+        assertEquals(ValueUpdate.ValueUpdateClassID.ASSIGN, serAssign.getValueUpdateClassID());
+        assertEquals(new IntegerFieldValue(4), serAssign.getValue());
+
+        FieldUpdate serAddFU = upd.getFieldUpdate(2);
+        assertEquals(type.getField("arrayoffloatfield"), serAddFU.getField());
+        ValueUpdate serAdd1 = serAddFU.getValueUpdate(0);
+        assertEquals(ValueUpdate.ValueUpdateClassID.ADD, serAdd1.getValueUpdateClassID());
+        FloatFieldValue addParam1 = (FloatFieldValue)serAdd1.getValue();
+        assertEquals(new FloatFieldValue(5.00f), addParam1);
+        ValueUpdate serAdd2 = serAddFU.getValueUpdate(1);
+        assertEquals(ValueUpdate.ValueUpdateClassID.ADD, serAdd2.getValueUpdateClassID());
+        FloatFieldValue addparam2 = (FloatFieldValue)serAdd2.getValue();
+        assertEquals(new FloatFieldValue(4.23f), addparam2);
+        ValueUpdate serAdd3 = serAddFU.getValueUpdate(2);
+        assertEquals(ValueUpdate.ValueUpdateClassID.ADD, serAdd3.getValueUpdateClassID());
+        FloatFieldValue addparam3 = (FloatFieldValue)serAdd3.getValue();
+        assertEquals(new FloatFieldValue(-1.00f), addparam3);
+
+        FieldUpdate arithFU = upd.getFieldUpdate(3);
+        assertEquals(type.getField("intfield"), serAssignFU.getField());
+        ValueUpdate serArith = arithFU.getValueUpdate(0);
+        assertEquals(ValueUpdate.ValueUpdateClassID.ARITHMETIC, serArith.getValueUpdateClassID());
+
+        FieldUpdate wsetFU = upd.getFieldUpdate(4);
+        assertEquals(type.getField("wsfield"), wsetFU.getField());
+        assertEquals(2, wsetFU.size());
+        ValueUpdate mapUpd = wsetFU.getValueUpdate(0);
+        assertEquals(ValueUpdate.ValueUpdateClassID.MAP, mapUpd.getValueUpdateClassID());
+        mapUpd = wsetFU.getValueUpdate(1);
+        assertEquals(ValueUpdate.ValueUpdateClassID.MAP, mapUpd.getValueUpdateClassID());
+    }
+
+    public void testGenerateSerializedFile() throws IOException {
+        docMan = DocumentTestCase.setUpCppDocType();
+
+        DocumentType type = docMan.getDocumentType("serializetest");
+        DocumentUpdate upd = new DocumentUpdate(type, new DocumentId("doc:update:test"));
+        FieldUpdate serAssign = FieldUpdate.createAssign(type.getField("intfield"), new IntegerFieldValue(4));
+        upd.addFieldUpdate(serAssign);
+        FieldUpdate serClearField = FieldUpdate.createClearField(type.getField("floatfield"));
+        upd.addFieldUpdate(serClearField);
+        List<FloatFieldValue> arrayOfFloat = new ArrayList<>();
+        arrayOfFloat.add(new FloatFieldValue(5.00f));
+        arrayOfFloat.add(new FloatFieldValue(4.23f));
+        arrayOfFloat.add(new FloatFieldValue(-1.00f));
+        FieldUpdate serAdd = FieldUpdate.createAddAll(type.getField("arrayoffloatfield"), arrayOfFloat);
+        upd.addFieldUpdate(serAdd);
+
+        GrowableByteBuffer buf = new GrowableByteBuffer(100, 2.0f);
+        upd.serialize(buf);
+
+        int size = buf.position();
+        buf.position(0);
+
+        FileOutputStream fos = new FileOutputStream("src/tests/data/serializeupdatejava.dat");
+        fos.write(buf.array(), 0, size);
+        fos.close();
+    }
+
+    public void testRequireThatAddAllCombinesFieldUpdates() {
+        DocumentType docType = new DocumentType("my_type");
+        Field field = new Field("my_int", DataType.INT);
+        docType.addField(field);
+
+        FieldUpdate fooField = FieldUpdate.createAssign(field, new IntegerFieldValue(1));
+        DocumentUpdate fooUpdate = new DocumentUpdate(docType, new DocumentId("doc:foo:"));
+        fooUpdate.addFieldUpdate(fooField);
+
+        FieldUpdate barField = FieldUpdate.createAssign(field, new IntegerFieldValue(2));
+        DocumentUpdate barUpdate = new DocumentUpdate(docType, new DocumentId("doc:foo:"));
+        barUpdate.addFieldUpdate(barField);
+
+        fooUpdate.addAll(barUpdate);
+        assertEquals(1, fooUpdate.getFieldUpdates().size());
+        FieldUpdate fieldUpdate = fooUpdate.getFieldUpdate(0);
+        assertNotNull(fieldUpdate);
+        assertEquals(field, fieldUpdate.getField());
+        assertEquals(2, fieldUpdate.getValueUpdates().size());
+        ValueUpdate valueUpdate = fieldUpdate.getValueUpdate(0);
+        assertNotNull(valueUpdate);
+        assertTrue(valueUpdate instanceof AssignValueUpdate);
+        assertEquals(new IntegerFieldValue(1), valueUpdate.getValue());
+        assertNotNull(valueUpdate = fieldUpdate.getValueUpdate(1));
+        assertTrue(valueUpdate instanceof AssignValueUpdate);
+        assertEquals(new IntegerFieldValue(2), valueUpdate.getValue());
+    }
+
+    public void testInstantiationAndEqualsHashCode() {
+        DocumentType type = new DocumentType("doo");
+        DocumentUpdate d1 = new DocumentUpdate(type, new DocumentId("doc:this:is:a:test"));
+        DocumentUpdate d2 = new DocumentUpdate(type, "doc:this:is:a:test");
+
+        assertEquals(d1, d2);
+        assertEquals(d1, d1);
+        assertEquals(d2, d1);
+        assertEquals(d2, d2);
+        assertFalse(d1.equals(new Object()));
+        assertEquals(d1.hashCode(), d2.hashCode());
+    }
+
+    public void testThatApplyingToWrongTypeFails() {
+        DocumentType t1 = new DocumentType("doo");
+        DocumentUpdate documentUpdate = new DocumentUpdate(t1, new DocumentId("doc:this:is:a:test"));
+
+        DocumentType t2 = new DocumentType("foo");
+        Document document = new Document(t2, "doc:this:is:another:test");
+
+        try {
+            documentUpdate.applyTo(document);
+            fail("Should have gotten exception here.");
+        } catch (IllegalArgumentException iae) {
+            //ok!
+        }
+    }
+
+    public void testFieldUpdatesInDocUp() throws ParseException {
+        DocumentType t1 = new DocumentType("doo");
+        Field f1 = new Field("field1", DataType.STRING);
+        Field f2 = new Field("field2", DataType.STRING);
+        t1.addField(f1);
+        t1.addField(f2);
+
+        DocumentUpdate documentUpdate = new DocumentUpdate(t1, new DocumentId("doc:this:is:a:test"));
+
+        assertEquals(0, documentUpdate.size());
+
+        FieldUpdate fu1 = FieldUpdate.createAssign(f1, new StringFieldValue("banana"));
+        FieldUpdate fu2 = FieldUpdate.createAssign(f2, new StringFieldValue("apple"));
+        documentUpdate.addFieldUpdate(fu1);
+
+        assertEquals(1, documentUpdate.size());
+
+        documentUpdate.clearFieldUpdates();
+
+        assertEquals(0, documentUpdate.size());
+
+        documentUpdate.addFieldUpdate(fu1);
+        documentUpdate.addFieldUpdate(fu2);
+
+        assertEquals(2, documentUpdate.size());
+
+
+        assertSame(fu1, documentUpdate.getFieldUpdate(f1));
+        assertSame(fu1, documentUpdate.getFieldUpdate(0));
+        assertSame(fu2, documentUpdate.getFieldUpdate(1));
+
+        documentUpdate.setFieldUpdate(0, fu2);
+        documentUpdate.setFieldUpdate(1, fu1);
+        assertEquals(2, documentUpdate.size());
+        assertSame(fu1, documentUpdate.getFieldUpdate(1));
+        assertSame(fu2, documentUpdate.getFieldUpdate(0));
+
+        try {
+            documentUpdate.setFieldUpdates(null);
+            fail("Should have gotten NullPointerException");
+        } catch (NullPointerException npe) {
+            //ok!
+        }
+
+        List<FieldUpdate> fus = new ArrayList<>();
+        fus.add(fu1);
+        fus.add(fu2);
+
+        documentUpdate.setFieldUpdates(fus);
+        assertEquals(2, documentUpdate.size());
+        assertSame(fu1, documentUpdate.getFieldUpdate(0));
+        assertSame(fu2, documentUpdate.getFieldUpdate(1));
+
+        documentUpdate.removeFieldUpdate(1);
+        assertEquals(1, documentUpdate.size());
+        assertSame(fu1, documentUpdate.getFieldUpdate(0));
+
+
+        documentUpdate.toString();
+
+        assertFalse(documentUpdate.isEmpty());
+
+        Iterator<FieldPathUpdate> fpUpdates = documentUpdate.iterator();
+        assertFalse(fpUpdates.hasNext());
+    }
+
+    public void testAddAll() {
+        DocumentType t1 = new DocumentType("doo");
+        DocumentType t2 = new DocumentType("foo");
+        Field f1 = new Field("field1", DataType.STRING);
+        Field f2 = new Field("field2", DataType.STRING);
+        t1.addField(f1);
+        t2.addField(f2);
+
+        DocumentUpdate du1 = new DocumentUpdate(t1, new DocumentId("doc:this:is:a:test"));
+        DocumentUpdate du2 = new DocumentUpdate(t2, "doc:this:is:another:test");
+
+        FieldUpdate fu1 = FieldUpdate.createAssign(f1, new StringFieldValue("banana"));
+        FieldUpdate fu2 = FieldUpdate.createAssign(f2, new StringFieldValue("apple"));
+        du1.addFieldUpdate(fu1);
+        du2.addFieldUpdate(fu2);
+
+        du1.addAll(null);
+
+        try {
+            du1.addAll(du2);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {
+            //ok!
+        }
+
+        DocumentUpdate du3 = new DocumentUpdate(t2, new DocumentId("doc:this:is:a:test"));
+
+        try {
+            du1.addAll(du3);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {
+            //ok!
+        }
+    }
+
+    private void assertDocumentUpdateFlag(boolean createIfNonExistent, int value) {
+        DocumentUpdateFlags f1 = new DocumentUpdateFlags();
+        f1.setCreateIfNonExistent(createIfNonExistent);
+        assertEquals(createIfNonExistent, f1.getCreateIfNonExistent());
+        int combined = f1.injectInto(value);
+        System.out.println("createIfNonExistent=" + createIfNonExistent + ", value=" + value + ", combined=" + combined);
+
+        DocumentUpdateFlags f2 = DocumentUpdateFlags.extractFlags(combined);
+        int extractedValue = DocumentUpdateFlags.extractValue(combined);
+        assertEquals(createIfNonExistent, f2.getCreateIfNonExistent());
+        assertEquals(value, extractedValue);
+    }
+
+    public void testRequireThatDocumentUpdateFlagsIsWorking() {
+        { // create-if-non-existent = true
+            assertDocumentUpdateFlag(true, 0);
+            assertDocumentUpdateFlag(true, 1);
+            assertDocumentUpdateFlag(true, 2);
+            assertDocumentUpdateFlag(true, 9999);
+            assertDocumentUpdateFlag(true, 0xFFFFFFE);
+            assertDocumentUpdateFlag(true, 0xFFFFFFF);
+        }
+        { // create-if-non-existent = false
+            assertDocumentUpdateFlag(false, 0);
+            assertDocumentUpdateFlag(false, 1);
+            assertDocumentUpdateFlag(false, 2);
+            assertDocumentUpdateFlag(false, 9999);
+            assertDocumentUpdateFlag(false, 0xFFFFFFE);
+            assertDocumentUpdateFlag(false, 0xFFFFFFF);
+        }
+    }
+
+    public void testRequireThatCreateIfNonExistentFlagIsSerializedAndDeserialized() {
+        docUp.setCreateIfNonExistent(true);
+
+        DocumentSerializer serializer = DocumentSerializerFactory.createHead(new GrowableByteBuffer());
+        docUp.serialize(serializer);
+        serializer.getBuf().flip();
+
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.createHead(docMan, serializer.getBuf());
+        DocumentUpdate deserialized = new DocumentUpdate(deserializer);
+        assertEquals(docUp, deserialized);
+        assertTrue(deserialized.getCreateIfNonExistent());
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/DocumentUtilTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUtilTestCase.java
new file mode 100644
index 00000000000..60d4e994bff
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/DocumentUtilTestCase.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class DocumentUtilTestCase extends junit.framework.TestCase {
+
+    public void testBasic() {
+/***
+        final long heapBytes = Runtime.getRuntime().maxMemory();
+        long maxPendingBytes = DocumentUtil.calculateMaxPendingSize(1.0, 1.0, 0);
+        almostEquals(heapBytes / 5, maxPendingBytes, 512*1024);
+    }
+
+    public void test2_2() {
+        final long heapBytes = Runtime.getRuntime().maxMemory();
+        long maxPendingBytes = DocumentUtil.calculateMaxPendingSize(2.0, 2.0, 0);
+        almostEquals(heapBytes / 5, maxPendingBytes, 512*1024);
+    }
+
+    public void test4_4() {
+        final long heapBytes = Runtime.getRuntime().maxMemory();
+        long maxPendingBytes = DocumentUtil.calculateMaxPendingSize(4.0, 4.0, 0);
+        almostEquals(heapBytes / 17, maxPendingBytes, 512*1024);
+    }
+
+    public void test8_8() {
+        final long heapBytes = Runtime.getRuntime().maxMemory();
+        long maxPendingBytes = DocumentUtil.calculateMaxPendingSize(8.0, 8.0, 0);
+        almostEquals(heapBytes / 65, maxPendingBytes, 512*1024);
+    }
+
+    public void test10000_10000() {
+        long maxPendingBytes = DocumentUtil.calculateMaxPendingSize(10000.0, 10000.0, 0);
+        almostEquals(1*1024*1024, maxPendingBytes, 512*1024);
+***/
+    }
+
+    private static void almostEquals(long expected, long actual, long off) {
+        System.err.println("Got actual " + (((double) actual) / 1024d / 1024d) + " MB, expected "
+                + (((double) expected) / 1024d / 1024d) + " MB, within +/- " + (((double) off) / 1024d / 1024d) + " MB");
+
+        assertTrue(actual > (expected - off) && actual < (expected + off));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/FieldPathEntryTestCase.java b/document/src/test/java/com/yahoo/document/FieldPathEntryTestCase.java
new file mode 100644
index 00000000000..064cadc0db9
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/FieldPathEntryTestCase.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class FieldPathEntryTestCase {
+    @Test
+    public void testKeyParseResult() {
+        FieldPathEntry.KeyParseResult result1 = new FieldPathEntry.KeyParseResult("banana", 2);
+        FieldPathEntry.KeyParseResult result2 = new FieldPathEntry.KeyParseResult("banana", 2);
+        FieldPathEntry.KeyParseResult result3 = new FieldPathEntry.KeyParseResult("apple", 2);
+        FieldPathEntry.KeyParseResult result4 = new FieldPathEntry.KeyParseResult("banana", 3);
+
+
+        assertThat(result1, equalTo(result2));
+        assertThat(result2, equalTo(result1));
+        assertThat(result1.hashCode(), equalTo(result2.hashCode()));
+        assertThat(result1.toString(), equalTo(result2.toString()));
+
+        assertThat(result1, not(equalTo(result3)));
+        assertThat(result3, not(equalTo(result1)));
+        assertThat(result1.hashCode(), not(equalTo(result3.hashCode())));
+        assertThat(result1.toString(), not(equalTo(result3.toString())));
+
+        assertThat(result1, not(equalTo(result4)));
+        assertThat(result4, not(equalTo(result1)));
+        assertThat(result1.hashCode(), not(equalTo(result4.hashCode())));
+        assertThat(result1.toString(), not(equalTo(result4.toString())));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/FieldTestCase.java b/document/src/test/java/com/yahoo/document/FieldTestCase.java
new file mode 100644
index 00000000000..b8be9e85813
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/FieldTestCase.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+/**
+ * @author <a href="thomasg@yahoo-inc.com>Thomas Gundersen</a>
+ */
+public class FieldTestCase extends junit.framework.TestCase {
+    public FieldTestCase(String name) {
+        super(name);
+    }
+
+    public void testIdSettingConflict() {
+        DocumentType doc = new DocumentType("testdoc");
+        Field one = doc.addField("one", DataType.STRING);
+        one.setId(60, doc);
+
+        Field two = doc.addField("two", DataType.STRING);
+        two.setId(61, doc);
+
+        try {
+            Field three = doc.addField("three", DataType.STRING);
+            three.setId(60, doc);
+            fail("Allowed to set duplicate id");
+        } catch (IllegalArgumentException e) {
+            // Success
+        }
+    }
+
+    public void testSettingReservedId() {
+        DocumentType doc = new DocumentType("testdoc");
+        try {
+            Field one = doc.addField("one", DataType.STRING);
+            one.setId(127, doc);
+            fail("Allowed to set reserved id");
+        } catch (IllegalArgumentException e) {
+            // Success
+        }
+
+        try {
+            Field one = doc.addField("one", DataType.STRING);
+            one.setId(100, doc);
+            fail("Allowed to set reserved id");
+        } catch (IllegalArgumentException e) {
+            // Success
+        }
+
+        try {
+            Field one = doc.addField("one", DataType.STRING);
+            one.setId(-1, doc);
+            fail("Allowed to set reserved id");
+        } catch (IllegalArgumentException e) {
+            // Success
+        }
+        doc.removeField("one");
+        Field one = doc.addField("one", DataType.STRING);
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/GlobalIdTestCase.java b/document/src/test/java/com/yahoo/document/GlobalIdTestCase.java
new file mode 100644
index 00000000000..30b8f4c42ea
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/GlobalIdTestCase.java
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import java.util.Arrays;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class GlobalIdTestCase extends junit.framework.TestCase {
+    private final byte[] raw0 = new byte[0];
+    private final byte[] raw1_0 = new byte[]{(byte) 0};
+    private final byte[] raw2_11 = new byte[]{(byte) 1, (byte) 1};
+    private final byte[] raw2_minus1_1 = new byte[]{(byte) -1, (byte) 1};
+    private final byte[] raw12_1to12 = new byte[]{(byte) 1, (byte) 2, (byte) 3, (byte) 4, (byte) 5, (byte) 6, (byte) 7,
+            (byte) 8, (byte) 9, (byte) 10, (byte) 11, (byte) 12};
+    private final byte[] raw13 = new byte[]{(byte) 1, (byte) 2, (byte) 3, (byte) 4, (byte) 5, (byte) 6, (byte) 7,
+            (byte) 8, (byte) 9, (byte) 10, (byte) 11, (byte) 12, (byte) 13};
+
+    private final BucketIdFactory bucketIdFactory = new BucketIdFactory();
+
+    public void testRaw0() {
+        GlobalId gid = new GlobalId(raw0);
+        assertEquals(12, gid.getRawId().length);
+        byte[] raw = gid.getRawId();
+        for (byte b : raw) {
+            assertEquals((byte) 0, b);
+        }
+
+        GlobalId gid2 = new GlobalId(raw1_0);
+        assertEquals(12, gid2.getRawId().length);
+        byte[] raw2 = gid2.getRawId();
+        for (byte b : raw2) {
+            assertEquals((byte) 0, b);
+        }
+
+        assertEquals(gid, gid2);
+        assertTrue(Arrays.equals(raw, raw2));
+        assertEquals(gid.hashCode(), gid2.hashCode());
+    }
+
+    public void testLonger() {
+        GlobalId gid1 = new GlobalId(raw2_11);
+        GlobalId gid2 = new GlobalId(raw2_minus1_1);
+
+        assertFalse(gid1.equals(gid2));
+        assertFalse(gid1.hashCode() == gid2.hashCode());
+
+        GlobalId gid3 = new GlobalId(raw13);
+        GlobalId gid4 = new GlobalId(raw12_1to12);
+        assertEquals(gid3, gid4);
+        assertEquals(gid3.hashCode(), gid4.hashCode());
+    }
+
+    public void testCompareTo() {
+        GlobalId gid0 = new GlobalId(raw1_0);
+        GlobalId gid11 = new GlobalId(raw2_11);
+        GlobalId gidminus11 = new GlobalId(raw2_minus1_1);
+
+        assertEquals(-1, gid0.compareTo(gid11));
+        assertEquals(1, gid11.compareTo(gid0));
+
+        assertEquals(-1, gid0.compareTo(gidminus11));
+        assertEquals(1, gidminus11.compareTo(gid0));
+
+        assertEquals(-1, gid11.compareTo(gidminus11));
+        assertEquals(1, gidminus11.compareTo(gid11));
+    }
+
+    private void verifyGidToBucketIdMapping(String idString) {
+        DocumentId documentId = new DocumentId(idString);
+        GlobalId globalId = new GlobalId(documentId.getGlobalId());
+        BucketId bucketIdThroughGlobalId = globalId.toBucketId();
+        BucketId bucketIdThroughFactory = bucketIdFactory.getBucketId(documentId);
+        assertEquals(bucketIdThroughFactory, bucketIdThroughGlobalId);
+    }
+
+    public void testToBucketId() {
+        verifyGidToBucketIdMapping("userdoc:ns:1:abc");
+        verifyGidToBucketIdMapping("userdoc:ns:1000:abc");
+        verifyGidToBucketIdMapping("userdoc:hsgf:18446744073700000000:dfdfsdfg");
+        verifyGidToBucketIdMapping("groupdoc:ns:somegroup:hmm");
+        verifyGidToBucketIdMapping("doc:foo:test");
+        verifyGidToBucketIdMapping("doc:myns:http://foo.bar");
+        verifyGidToBucketIdMapping("doc:jsrthsdf:a234aleingzldkifvasdfgadf");
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/IdIdStringTest.java b/document/src/test/java/com/yahoo/document/IdIdStringTest.java
new file mode 100644
index 00000000000..bbdf45ec088
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/IdIdStringTest.java
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.idstring.IdIdString;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: magnarn
+ * Date: 10/16/12
+ * Time: 9:10 AM
+ */
+public class IdIdStringTest {
+    @Test
+    public void requireThatIdIdStringGeneratesProperString() throws Exception {
+        DocumentId docId = new DocumentId(new IdIdString("namespace", "type", "g=group", "foobar"));
+        assertEquals("id:namespace:type:g=group:foobar", docId.toString());
+    }
+
+    @Test
+    public void requireThatEmptyKeyValuesAreOk() throws Exception {
+        DocumentId docId = new DocumentId(new IdIdString("namespace", "type", "", "foobar"));
+        assertEquals("id:namespace:type::foobar", docId.toString());
+    }
+
+    @Test
+    public void requireThatIdIdStringCanBehaveLikeGroupDoc() throws Exception {
+        DocumentId docId1 = new DocumentId(new IdIdString("namespace", "type", "g=foo", "foo"));
+        DocumentId docId2 = new DocumentId(new IdIdString("namespace", "type", "g=foo", "bar"));
+        DocumentId docId3 = new DocumentId(new IdIdString("namespace", "type", "g=bar", "baz"));
+        assertEquals(docId1.getScheme().getLocation(), docId2.getScheme().getLocation());
+        assert(docId1.getScheme().getLocation() != docId3.getScheme().getLocation());
+    }
+
+    @Test
+    public void requireThatIdIdStringCanBehaveLikeUserDoc() throws Exception {
+        DocumentId docId1 = new DocumentId(new IdIdString("namespace", "type", "n=10", "foo"));
+        DocumentId docId2 = new DocumentId(new IdIdString("namespace", "type", "n=10", "bar"));
+        DocumentId docId3 = new DocumentId(new IdIdString("namespace", "type", "n=20", "baz"));
+        assertEquals(docId1.getScheme().getLocation(), docId2.getScheme().getLocation());
+        assert(docId1.getScheme().getLocation() != docId3.getScheme().getLocation());
+    }
+
+    @Test
+    public void requireThatIllegalKeyValuesThrow() throws Exception {
+        try {
+            new IdIdString("namespace", "type", "illegal=key", "foo");
+            fail();
+        } catch (IllegalArgumentException e) {
+        }
+    }
+
+    @Test
+    public void requireThatKeysWithoutValuesThrow() throws Exception {
+        try {
+            new IdIdString("namespace", "type", "illegal-pair", "foo");
+            fail();
+        } catch (IllegalArgumentException e) {
+        }
+    }
+
+    @Test
+    public void requireThatIdIdStringCanReplaceType() throws Exception {
+        String type = IdIdString.replaceType("id:namespace:type::foo", "newType");
+        assertEquals("id:namespace:newType::foo", type);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java b/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java
new file mode 100644
index 00000000000..9e0c4e353f2
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;

+

+import com.yahoo.document.datatypes.*;

+import org.junit.Before;

+import org.junit.Test;

+

+/**

+ * Tests for ticket 6394548

+ */

+public class IncompatibleFieldTypesTest {

+    private DataType arrayOfStrings;

+    private StructDataType struct;

+    private StructuredFieldValue root;

+

+    @Before

+    public void setUp() {

+        arrayOfStrings = new ArrayDataType(DataType.STRING);

+        struct = new StructDataType("fancypants");

+        struct.addField(new Field("stringarray", arrayOfStrings, false));

+        DataType weightedSetOfStrings = DataType.getWeightedSet(DataType.STRING, false, false);

+        struct.addField(new Field("stringws", weightedSetOfStrings, false));

+

+        root = struct.createFieldValue();

+        root.setFieldValue("stringarray", arrayOfStrings.createFieldValue());

+        root.setFieldValue("stringws", weightedSetOfStrings.createFieldValue());

+    }

+

+    @Test(expected = IllegalArgumentException.class)

+    public void testAddingIncompatibleFieldToArrayFails() {

+        System.out.println(root.getFieldValue("stringarray").getDataType().createFieldValue().getClass().getName());

+        System.out.println(root.getFieldValue("stringarray").getDataType().createFieldValue().getDataType().toString());

+

+        ((Array)root.getFieldValue("stringarray")).add(new IntegerFieldValue(1234));

+    }

+

+    @Test(expected = IllegalArgumentException.class)

+    public void testAddingIncompatibleFieldToWeightedSetFails() {

+        System.out.println(root.getFieldValue("stringws").getDataType().createFieldValue().getClass().getName());

+        System.out.println(root.getFieldValue("stringws").getDataType().createFieldValue().getDataType().toString());

+

+        ((WeightedSet<FieldValue>)root.getFieldValue("stringws")).put(new IntegerFieldValue(1234), 100);

+    }

+}

diff --git a/document/src/test/java/com/yahoo/document/NumericDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/NumericDataTypeTestCase.java
new file mode 100644
index 00000000000..ebd63aa47b5
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/NumericDataTypeTestCase.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.*;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class NumericDataTypeTestCase {
+
+    @Test
+    public void basic() {
+        NumericDataType type = new NumericDataType("foo", 0, FieldValue.class, IntegerFieldValue.getFactory());
+        NumericDataType clonedType = type.clone();
+        assertThat(type, equalTo(clonedType));
+        assertThat(type, not(sameInstance(clonedType)));
+    }
+
+    @Test
+    public void create() {
+        try {
+            new NumericDataType("foo", 0, IntegerFieldValue.class, IntegerFieldValue.getFactory()).createFieldValue(new Object());
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Class class java.lang.Object not applicable to an class " +
+                         "com.yahoo.document.datatypes.IntegerFieldValue instance.", e.getMessage());
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/PositionTypeTestCase.java b/document/src/test/java/com/yahoo/document/PositionTypeTestCase.java
new file mode 100644
index 00000000000..3fb9c8d3f9e
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/PositionTypeTestCase.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class PositionTypeTestCase {
+
+    @Test
+    public void requireThatPositionFactoryWorks() {
+        Struct val = PositionDataType.valueOf(6, 9);
+        assertEquals(new IntegerFieldValue(6), val.getFieldValue(PositionDataType.FIELD_X));
+        assertEquals(new IntegerFieldValue(9), val.getFieldValue(PositionDataType.FIELD_Y));
+        assertEquals(2, val.getFieldCount());
+
+        val = PositionDataType.fromLong((6L << 32) + 9);
+        assertEquals(new IntegerFieldValue(6), val.getFieldValue(PositionDataType.FIELD_X));
+        assertEquals(new IntegerFieldValue(9), val.getFieldValue(PositionDataType.FIELD_Y));
+        assertEquals(2, val.getFieldCount());
+    }
+
+    @Test
+    public void requireThatAccessorsWork() {
+        Struct val = PositionDataType.valueOf(6, 9);
+        assertEquals(new IntegerFieldValue(6), PositionDataType.getXValue(val));
+        assertEquals(new IntegerFieldValue(9), PositionDataType.getYValue(val));
+    }
+
+    @Test
+    public void requireThatConstantsMatchCpp() {
+        assertEquals("position", PositionDataType.STRUCT_NAME);
+        assertEquals("x", PositionDataType.FIELD_X);
+        assertEquals("y", PositionDataType.FIELD_Y);
+        assertEquals("foo_zcurve", PositionDataType.getZCurveFieldName("foo"));
+        assertEquals("foo.position", PositionDataType.getPositionSummaryFieldName("foo"));
+        assertEquals("foo.distance", PositionDataType.getDistanceSummaryFieldName("foo"));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/SimpleDocumentTestCase.java b/document/src/test/java/com/yahoo/document/SimpleDocumentTestCase.java
new file mode 100644
index 00000000000..853563165e4
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/SimpleDocumentTestCase.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SimpleDocumentTestCase {
+
+    @Test
+    public void requireThatAccessorsWorks() {
+        DocumentType type = new DocumentType("test");
+        type.addField("int", DataType.INT);
+        Document doc = new Document(type, "doc:scheme:");
+        SimpleDocument simple = new SimpleDocument(doc);
+
+        assertNull(simple.get("int"));
+        assertNull(doc.getFieldValue("int"));
+
+        simple.set("int", 69);
+        assertEquals(69, simple.get("int"));
+        assertEquals(new IntegerFieldValue(69), doc.getFieldValue("int"));
+
+        simple.remove("int");
+        assertNull(simple.get("int"));
+        assertNull(doc.getFieldValue("int"));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/StructDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/StructDataTypeTestCase.java
new file mode 100755
index 00000000000..6e5bfbb56d4
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/StructDataTypeTestCase.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.Struct;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class StructDataTypeTestCase extends junit.framework.TestCase {
+    public void testSimpleInheritance() {
+        StructDataType personType = new StructDataType("person");
+        Field firstName = new Field("firstname", DataType.STRING);
+        Field lastName = new Field("lastname", DataType.STRING);
+        personType.addField(firstName);
+        personType.addField(lastName);
+
+        StructDataType employeeType = new StructDataType("employee");
+        Field empId = new Field("employeeid", DataType.INT);
+        employeeType.addField(empId);
+
+        assertEquals(2, personType.getFieldCount());
+        assertEquals("firstname", personType.getFields().toArray(new Field[0])[0].getName());
+        assertEquals("lastname", personType.getFields().toArray(new Field[0])[1].getName());
+        assertEquals(1, employeeType.getFieldCount());
+        assertEquals("employeeid", employeeType.getFields().toArray(new Field[0])[0].getName());
+
+        employeeType.inherit(personType);
+
+        assertEquals(2, personType.getFieldCount());
+        assertEquals("firstname", personType.getFields().toArray(new Field[0])[0].getName());
+        assertEquals("lastname", personType.getFields().toArray(new Field[0])[1].getName());
+        assertEquals(3, employeeType.getFieldCount());
+        assertEquals("employeeid", employeeType.getFields().toArray(new Field[0])[0].getName());
+        assertEquals("firstname", employeeType.getFields().toArray(new Field[0])[1].getName());
+        assertEquals("lastname", employeeType.getFields().toArray(new Field[0])[2].getName());
+    }
+
+    public void testCompatibleWith() {
+        StructDataType personType = new StructDataType("person");
+        Field firstName = new Field("firstname", DataType.STRING);
+        Field lastName = new Field("lastname", DataType.STRING);
+        personType.addField(firstName);
+        personType.addField(lastName);
+
+        StructDataType employeeType = new StructDataType("employee");
+        Field empId = new Field("employeeid", DataType.INT);
+        employeeType.addField(empId);
+        employeeType.inherit(personType);
+
+        Struct person = new Struct(personType);
+        Struct employee = new Struct(employeeType);
+
+        assertTrue(personType.isValueCompatible(person));
+        assertTrue(personType.isValueCompatible(employee));
+
+        assertTrue(employeeType.isValueCompatible(employee));
+        assertFalse(employeeType.isValueCompatible(person));
+
+        StructDataType containerType = new StructDataType("containerstruct");
+        Field structPolymorphic = new Field("structpolymorphic", personType);
+        containerType.addField(structPolymorphic);
+
+        Struct container = new Struct(containerType);
+        container.setFieldValue(structPolymorphic, person);
+        container.setFieldValue(structPolymorphic, employee);
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java
new file mode 100644
index 00000000000..6f841aac821
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class TemporaryDataTypeTestCase {
+    @Test
+    public void requireNulls() {
+        TemporaryDataType type = new TemporaryDataType(0);
+        assertThat(type.createFieldValue(new Object()), nullValue());
+        assertThat(type.createFieldValue(), nullValue());
+        assertThat(type.getValueClass(), nullValue());
+        assertThat(type.isValueCompatible(new StringFieldValue("")), is(false));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/TemporaryStructuredDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/TemporaryStructuredDataTypeTestCase.java
new file mode 100644
index 00000000000..aad0505c365
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/TemporaryStructuredDataTypeTestCase.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class TemporaryStructuredDataTypeTestCase {
+    @Test
+    public void basic() {
+        TemporaryStructuredDataType type = TemporaryStructuredDataType.create("banana");
+        assertThat(type.getName(), equalTo("banana"));
+        int originalId = type.getId();
+        type.setName("apple");
+        assertThat(type.getName(), equalTo("apple"));
+        assertThat(originalId, not(equalTo(type.getId())));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/AbstractTypesTest.java b/document/src/test/java/com/yahoo/document/annotation/AbstractTypesTest.java
new file mode 100755
index 00000000000..cc9cb8b2dd2
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/AbstractTypesTest.java
@@ -0,0 +1,150 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public abstract class AbstractTypesTest {
+
+    protected DocumentTypeManager man;
+    protected StructDataType person;
+    protected AnnotationReferenceDataType personReference;
+    protected StructDataType relative;
+    protected AnnotationType dummy;
+    protected AnnotationType number;
+    protected AnnotationType personA;
+    protected AnnotationType relativeA;
+    protected AnnotationType banana;
+    protected AnnotationType apple;
+    protected AnnotationType grape;
+    protected DocumentType docType;
+
+    public AbstractTypesTest() {
+        man = new DocumentTypeManager();
+
+        person = new StructDataType("person");
+        person.addField(new Field("firstname", DataType.STRING));
+        person.addField(new Field("lastname", DataType.STRING));
+        person.addField(new Field("birthyear", DataType.INT));
+        man.register(person);
+
+        personA = new AnnotationType("person", person);
+        man.getAnnotationTypeRegistry().register(personA);
+
+        personReference = new AnnotationReferenceDataType(personA);
+        man.register(personReference);
+
+        relative = new StructDataType("relative");
+        relative.addField(new Field("title", DataType.STRING));
+        relative.addField(new Field("related", personReference));
+        man.register(relative);
+
+        dummy = new AnnotationType("dummy");
+        number = new AnnotationType("number", DataType.INT);
+        relativeA = new AnnotationType("relative", relative);
+        banana = new AnnotationType("banana");
+        apple = new AnnotationType("apple");
+        grape = new AnnotationType("grape");
+        man.getAnnotationTypeRegistry().register(dummy);
+        man.getAnnotationTypeRegistry().register(number);
+        man.getAnnotationTypeRegistry().register(relativeA);
+        man.getAnnotationTypeRegistry().register(banana);
+        man.getAnnotationTypeRegistry().register(apple);
+        man.getAnnotationTypeRegistry().register(grape);
+
+        docType = new DocumentType("dokk");
+        docType.addField("age", DataType.BYTE);
+        docType.addField("story", DataType.STRING);
+        docType.addField("date", DataType.INT);
+        docType.addField("friend", DataType.LONG);
+        man.register(docType);
+    }
+
+    protected StringFieldValue getAnnotatedString() {
+        StringFieldValue text = new StringFieldValue("help me help me i'm stuck inside a computer!");
+        {
+            AlternateSpanList alternateSpanList = new AlternateSpanList();
+            SpanTree tree = new SpanTree("ballooo", alternateSpanList);
+            text.setSpanTree(tree);
+
+            Span s1 = new Span(1, 2);
+            Span s2 = new Span(3, 4);
+            Span s3 = new Span(4, 5);
+            alternateSpanList.add(s1).add(s2).add(s3);
+            AlternateSpanList s4 = new AlternateSpanList();
+            Span s5 = new Span(7, 8);
+            Span s6 = new Span(8, 9);
+            s4.add(s5).add(s6);
+            alternateSpanList.add(s4);
+
+            tree.annotate(s2, dummy);
+            tree.annotate(s2, new Annotation(number, new IntegerFieldValue(1234)));
+
+            Struct mother = new Struct(person);
+            mother.setFieldValue("firstname", "jenny");
+            mother.setFieldValue("lastname", "olsen");
+            mother.setFieldValue("birthyear", 1909);
+            Annotation motherA = new Annotation(personA, mother);
+            tree.annotate(s2, motherA);
+
+            Struct daughter = new Struct(relative);
+            daughter.setFieldValue("title", "daughter");
+            daughter.setFieldValue("related", new AnnotationReference(personReference, motherA));
+            tree.annotate(s6, new Annotation(relativeA, daughter));
+
+            tree.annotate(s1, dummy);
+            tree.annotate(s3, dummy);
+            tree.annotate(s3, new Annotation(number, new IntegerFieldValue(2344)));
+            tree.annotate(s5, dummy);
+
+            List<SpanNode> alternateChildren = new ArrayList<>();
+            Span s7 = new Span(1, 4);
+            Span s8 = new Span(1, 9);
+            Span s9 = new Span(5, 6);
+            alternateChildren.add(s7);
+            alternateChildren.add(s8);
+            alternateChildren.add(s9);
+
+            alternateSpanList.addChildren(alternateChildren, 5.55);
+        }
+        {
+            SpanList root = new SpanList();
+            SpanTree tree = new SpanTree("fruits", root);
+            text.setSpanTree(tree);
+
+
+            Span s1 = new Span(5, 6);
+            Span s2 = new Span(11, 3);
+            Span s3 = new Span(14, 7);
+            root.add(s1).add(s2).add(s3);
+
+            tree.annotate(s1, banana);
+            tree.annotate(s1, grape);
+            tree.annotate(s2, banana);
+            tree.annotate(s3, apple);
+            tree.annotate(s3, grape);
+            tree.annotate(s3, grape);
+
+            SpanList s4 = new SpanList();
+            Span s5 = new Span(23,1);
+            s4.add(s5);
+            root.add(s4);
+
+            tree.annotate(s4, grape);
+            tree.annotate(s5, apple);
+        }
+        return text;
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListAdvTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListAdvTestCase.java
new file mode 100644
index 00000000000..e5950abe244
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListAdvTestCase.java
@@ -0,0 +1,429 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.TreeSet;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:mpraveen@yahoo-inc.com">Praveen Mohan</a>
+ *
+ * This test covers all possible scenarios in AlternateSpanList.
+ * If you really want to debug, just turn on the debug flag to true.
+ *
+ */
+public class AlternateSpanListAdvTestCase {
+
+    private AnnotationType at1 = new AnnotationType("person", DataType.STRING);
+    private AnnotationType at2 = new AnnotationType("street", DataType.STRING);
+    private AnnotationType at3 = new AnnotationType("city", DataType.STRING);
+
+    private StringFieldValue fv1 = (StringFieldValue) at1.getDataType().createFieldValue();
+    private StringFieldValue fv2 = (StringFieldValue) at2.getDataType().createFieldValue();
+    private StringFieldValue fv3 = (StringFieldValue) at3.getDataType().createFieldValue();
+    private StringFieldValue fv11 = (StringFieldValue) at1.getDataType().createFieldValue();
+    private StringFieldValue fv22 = (StringFieldValue) at2.getDataType().createFieldValue();
+    private StringFieldValue fv33 = (StringFieldValue) at3.getDataType().createFieldValue();
+    private StringFieldValue fv111 = (StringFieldValue) at1.getDataType().createFieldValue();
+    private StringFieldValue fv222 = (StringFieldValue) at1.getDataType().createFieldValue();
+
+    private SpanList root;
+    private SpanTree tree;
+    private SpanNode span1, span2, span3;
+    private SpanNode span11, span22, span33;
+    private SpanList alternate1, alternate2, branch0;
+    private List<SpanNode> subtreeList1, subtreeList2;
+    private AlternateSpanList branch;
+    private AlternateSpanList branch3, branch2;
+    private Annotation an1;
+
+    private boolean debug = false;
+
+    @Before
+    public void buildTree_List() {
+        root = new SpanList();
+        tree = new SpanTree("test", root);
+        branch = new AlternateSpanList();
+        span1 = new Span(0, 3);
+        span2 = new Span(1, 9);
+        span3 = new Span(12, 10);
+
+        span11 = new Span(0, 3);
+        span22 = new Span(1, 9);
+        span33 = new Span(12, 10);
+
+        an1 = new Annotation(at1, fv1);
+        Annotation an2 = new Annotation(at2, fv2);
+        Annotation an3 = new Annotation(at3, fv3);
+        Annotation an11 = new Annotation(at1, fv11);
+        Annotation an22 = new Annotation(at2, fv22);
+        Annotation an33 = new Annotation(at3, fv33);
+
+        alternate1 = new SpanList();
+        alternate1.add(span3);
+        alternate1.add(span2);
+        alternate1.add(span1);
+
+        alternate2 = new SpanList();
+        alternate2.add(span11);
+        alternate2.add(span22);
+        alternate2.add(span33);
+
+        tree.annotate(span1, an1);
+        tree.annotate(span2, an2);
+        tree.annotate(span3, an3);
+
+        tree.annotate(span11, an11);
+        tree.annotate(span22, an22);
+        tree.annotate(span33, an33);
+
+        subtreeList1 = new ArrayList<SpanNode>();
+        subtreeList1.add(alternate1);
+
+        subtreeList2 = new ArrayList<SpanNode>();
+        subtreeList2.add(alternate2);
+        branch.clearChildren();
+        branch.addChildren(1, subtreeList1, 20.0d);
+        branch.addChildren(2, subtreeList2, 50.0d);
+
+        root.add(branch);
+    }
+
+    @Test(expected = IndexOutOfBoundsException.class)
+    public void assertInvalidGetFrom() {
+        assertEquals(0, branch.getFrom(50));
+    }
+
+    @Test (expected = IllegalStateException.class)
+    public void assertSharingAnnotationInstance() {
+        SpanNode testNode = new Span(0, 2);
+        tree.annotate(testNode, an1);
+    }
+
+    @Test (expected = IllegalStateException.class)
+    public void assertSharingSpanTreeRoot() {
+        tree = new SpanTree("dummy", root);
+    }
+
+
+    @Test (expected = IllegalStateException.class)
+    public void assertAddSameNodeTwice() {
+        root.add(branch);
+    }
+
+    @Test (expected = IndexOutOfBoundsException.class)
+    public void assertInvalidAdd() {
+        SpanNode newNode = new Span(10, 10);
+        branch.add(branch.getNumSubTrees(), newNode);
+    }
+
+    @Test (expected = IndexOutOfBoundsException.class)
+    public void assertInvalidChildIteratorIndex() {
+        branch.childIterator(branch.getNumSubTrees());
+    }
+
+    @Test (expected = IllegalStateException.class)
+    public void assertReuseRemovedNode() {
+        alternate1.remove(span1);
+        branch.add(span1);
+    }
+
+    @Test
+    public void assertTree_NodeSet1() {
+        if (debug) consumeAnnotations(tree, root);
+        assertEquals(-1, root.getFrom());
+        assertEquals(-1, root.getTo());
+        assertEquals(-1, branch.getFrom());
+        assertEquals(-1, branch.getTo());
+        assertEquals(0, branch.getFrom(1));
+        assertEquals(22, branch.getTo(1));
+        assertEquals(0, branch.getFrom(2));
+        assertEquals(22, branch.getTo(2));
+        assertEquals(3, branch.getNumSubTrees());
+        int no = branch.getNumSubTrees();
+
+        TreeSet<Double> set = new TreeSet<Double>();
+        for (int i = 0; i < no; i ++) {
+            double prob = branch.getProbability(i);
+            set.add(prob);
+        }
+
+        branch.sortSubTreesByProbability();
+
+        Iterator<Double> iter = set.descendingIterator();
+        for (int i = 0; i < no; i ++) {
+            double prob = branch.getProbability(i);
+            double prob1 = iter.next();
+            assertTrue(prob == prob1);
+        }
+        branch.normalizeProbabilities();
+        double highest = 0;
+        for (int i = 0; i < no; i ++) {
+            double prob = branch.getProbability(i);
+            if (i == 0) {
+                highest = prob;
+                continue;
+            }
+            assertFalse(prob >= highest || prob > 1.0);
+            highest = prob;
+        }
+
+        ListIterator<SpanNode> it = branch.childIterator();
+        assertSame(alternate2, it.next());
+        assertSame(alternate1, it.next());
+        assertFalse(it.hasNext());
+
+        SpanNode sn;
+
+        it = branch.childIteratorRecursive();
+        assertSame(span11, it.next());
+        assertSame(span22, it.next());
+        assertSame(span33, it.next());
+        assertSame(alternate2, it.next());
+        assertSame(span3, it.next());
+        assertSame(span2, it.next());
+        assertSame(span1, it.next());
+        assertSame(alternate1, it.next());
+        assertFalse(it.hasNext());
+
+
+        it = branch.childIterator(1);
+        assertSame(alternate1, it.next());
+        assertFalse(it.hasNext());
+
+
+        SpanNode snNew = new Span(15, 20);
+        List<SpanNode> alternate3 = new ArrayList<SpanNode>();
+        alternate3.add(snNew);
+        List<SpanNode> l = branch.setChildren(0, alternate3, 200.0d);
+        assertFalse (l.get(0) != alternate2);
+        assertFalse (branch.getProbability(0) != 200.0);
+
+        String s = branch.toString();
+        tree.cleanup();
+        boolean dMode = debug;
+        debug = false;
+        consumeAnnotations(tree, root);
+        debug = dMode;
+
+        branch.setProbability(0, 125.0d);
+        branch.setProbability(1, 75.0d);
+        branch.setProbability(2, 25.0d);
+        branch.sortSubTreesByProbability();
+        double initial = 125.0;
+        for (int j = 0; j < no; j++) {
+            double prob = branch.getProbability(j);
+            assertFalse (prob != initial);
+            initial = initial - 50.0d;
+        }
+        no = branch.getNumSubTrees();
+
+        for (int j = 0; j < no; no --) {
+            branch.removeChildren(j);
+        }
+
+        assertFalse (branch.getNumSubTrees() != 1);
+        assertFalse(branch.getProbability(0) != 1.0);
+        branch.addChildren(1, subtreeList1, 20.0d);
+        branch.addChildren(2, subtreeList2, 50.0d);
+
+        no = branch.getNumSubTrees();
+        assertFalse (no != 3);
+        branch.clearChildren();
+        assertFalse (branch.getNumSubTrees() != 3);
+        assertFalse(branch.getProbability(0) != 1.0);
+
+        branch.addChildren(1, subtreeList1, 20.0d);
+        branch.addChildren(2, subtreeList2, 50.0d);
+        no = branch.getNumSubTrees();
+        assertFalse (no != 5);
+        branch.clearChildren(1);
+        assertTrue (branch.getNumSubTrees() == no);
+        assertEquals(branch.getFrom(1), -1);
+        assertEquals(branch.getTo(1), -1);
+        assertTrue(branch.getProbability(1) == 20.0);
+
+        ListIterator<SpanNode> lit = branch.childIteratorRecursive(1);
+        assertFalse(lit.hasNext());
+        SpanNode newNode = new Span(10, 10);
+        branch.add(0, newNode);
+        lit = branch.childIteratorRecursive();
+        assertTrue(lit.hasNext());
+        assertFalse(lit.next() != newNode);
+
+        branch.removeChildren(1);
+        assertTrue (branch.getNumSubTrees() == (no-1));
+
+        branch.removeChildren();
+        no = branch.getNumSubTrees();
+        assertTrue (no == 1);
+        assertTrue (branch.getProbability(0) == 1.0);
+        assertEquals(branch.getFrom(), -1);
+        assertEquals(branch.getTo(), -1);
+
+        buildTree_List();
+
+        CharSequence fieldValue = "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
+        String actual = branch.getText(1, fieldValue).toString();
+        String expected = "MNOPQRSTUVBCDEFGHIJABC";
+        assertEquals(actual, expected);
+    }
+
+    @Test
+    public void assertTree_NodeSet2() {
+        root = new SpanList();
+        tree = new SpanTree("test", root);
+        branch = new AlternateSpanList();
+
+        Annotation an111 = new Annotation(at1, fv111);
+        Annotation an222 = new Annotation(at2, fv222);
+
+        branch0 = new SpanList();
+        SpanNode span1 = new Span(0, 3);
+        SpanNode span2 = new Span(3, 3);
+        branch0.add(span1);
+        branch0.add(span2);
+        branch.add(branch0);
+
+        ArrayList<SpanNode> list1 = new ArrayList<SpanNode>();
+        branch2 = new AlternateSpanList();
+        SpanNode sn1 = new Span(6, 4);
+        SpanNode sn2 = new Span(10, 4);
+        branch2.add(sn1);
+        branch2.add(sn2);
+        list1.add(branch2);
+        branch.setProbability(0, 20.0d);
+        branch.addChildren(1, list1, 30.0d);
+
+        branch3 = new AlternateSpanList();
+        SpanNode sn3 = new Span(15, 5);
+        SpanNode sn4 = new Span(20, 5);
+        ArrayList<SpanNode> list2 = new ArrayList<SpanNode>();
+        list2.add(sn3);
+        list2.add(sn4);
+        branch3.addChildren(1, list2, 20.0d);
+        branch3.setProbability(0, 10.0d);
+
+        List<SpanNode> list3 = new ArrayList<SpanNode>();
+        list3.add(branch3);
+        branch2.addChildren(1, list3, 50.0d);
+        branch2.setProbability(0, 25.0d);
+
+        SpanNode sn5 = new Span(25, 3);
+        branch3.add(sn5);
+
+        root.add(branch);
+
+        // Never bother. Just for debugging.
+        if (debug) {
+            System.out.println("===========NodeSet2 ================");
+            consumeAnnotations(tree, root);
+        }
+
+        assertEquals(0, root.getFrom());
+        assertEquals(6, root.getTo());
+        assertEquals(0, branch.getFrom());
+        assertEquals(6, branch.getTo());
+        assertEquals(6, branch2.getFrom());
+        assertEquals(14, branch2.getTo());
+        assertEquals(25, branch2.getFrom(1));
+        assertEquals(28, branch2.getTo(1));
+        assertEquals(25, branch3.getFrom());
+        assertEquals(28, branch3.getTo());
+        assertEquals(15, branch3.getFrom(1));
+        assertEquals(25, branch3.getTo(1));
+        assertFalse ((branch.getNumSubTrees() != branch2.getNumSubTrees()) ||
+            (branch.getNumSubTrees() != branch3.getNumSubTrees()));
+
+        branch3.sortSubTreesByProbability();
+        assertEquals(15, branch3.getFrom());
+        assertEquals(25, branch3.getTo());
+        assertEquals(25, branch3.getFrom(1));
+        assertEquals(28, branch3.getTo(1));
+    }
+
+
+    @After
+    public void removeTree() {
+        tree = null;
+    }
+
+    public void consumeAnnotations(SpanTree tree, SpanList root) {
+        if (debug) System.out.println("\n\nSpanList: [" + root.getFrom() + ", " + root.getTo() + "] num Children: " + root.numChildren());
+        if (debug) System.out.println("-------------------");
+        Iterator<SpanNode> childIterator = root.childIterator();
+        while (childIterator.hasNext()) {
+            SpanNode node = childIterator.next();
+            if (debug) System.out.println("\n\nSpan Node (" + node + "): [" + node.getFrom() + ", " + node.getTo() + "] ");
+            if (node instanceof AlternateSpanList) {
+                parseAlternateLists(tree, (AlternateSpanList)node);
+                if (debug) System.out.println("---- Alternate SpanList complete ---");
+            } else if (node instanceof SpanList) {
+                if (debug) System.out.println("Encountered another span list");
+                SpanList spl = (SpanList) node;
+                ListIterator<SpanNode> lli = spl.childIterator();
+                while (lli.hasNext()) System.out.print(" " + lli.next() + " ");
+                consumeAnnotations(tree, (SpanList) node);
+            } else {
+                   if (debug) System.out.println("\nGetting annotations for this span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+                   getAnnotationsForNode(tree, node);
+            }
+        }
+        if (debug) System.out.println("\nGetting annotations for the SpanList itself : [" + root.getFrom() + ", " + root.getTo() + "] ");
+        getAnnotationsForNode(tree, root);
+    }
+
+    public void parseAlternateLists(SpanTree tree, AlternateSpanList aspl) {
+        int no = aspl.getNumSubTrees();
+        if (debug) System.out.println("Parsing Alternate span list. No of subtrees: " + no);
+        int ctr = 0;
+        while (ctr < no) {
+            if (debug) System.out.println("\nSubTree: " + ctr);
+            ListIterator<SpanNode> lIter = aspl.childIteratorRecursive(ctr);
+            while (lIter.hasNext()) {
+                SpanNode spnNode = lIter.next();
+                if (debug) System.out.println("Parsing span node: [" + spnNode.getFrom() + ", " + spnNode.getTo() + "] ");
+                if (spnNode instanceof AlternateSpanList) {
+                    if (debug) System.out.println("A child alternate span list found. Recursing");
+                    parseAlternateLists(tree, (AlternateSpanList)spnNode);
+                }
+
+                getAnnotationsForNode(tree, spnNode);
+            }
+            ctr ++;
+        }
+    }
+
+    public void getAnnotationsForNode(SpanTree tree, SpanNode node) {
+        Iterator<Annotation> iter = tree.iterator(node);
+        boolean annotationPresent = false;
+        while (iter.hasNext()) {
+            annotationPresent = true;
+            Annotation xx = iter.next();
+            StringFieldValue fValue = (StringFieldValue) xx.getFieldValue();
+            if (debug) System.out.println("Annotation: " + xx);
+            if (fValue == null) {
+                if (debug) System.out.println("Field Value is null");
+                return;
+            } else {
+                if (debug) System.out.println("Field Value: " + fValue.getString());
+            }
+        }
+        if (!annotationPresent) {
+            if (debug) System.out.println("****No annotations found for the span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListTestCase.java
new file mode 100755
index 00000000000..f6131a42e8a
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListTestCase.java
@@ -0,0 +1,250 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ListIterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AlternateSpanListTestCase extends AbstractTypesTest {
+
+    @Test
+    public void testSerializeDeserialize() {
+        {
+            AlternateSpanList alternateSpanList = new AlternateSpanList();
+            serializeAndAssert(alternateSpanList);
+        }
+        {
+            AlternateSpanList alternateSpanList = new AlternateSpanList();
+            Span s1 = new Span(1, 2);
+            Span s2 = new Span(3, 4);
+            Span s3 = new Span(4, 5);
+            alternateSpanList.add(s1).add(s2).add(s3);
+            AlternateSpanList s4 = new AlternateSpanList();
+            Span s5 = new Span(7, 8);
+            Span s6 = new Span(8, 9);
+            s4.add(s5).add(s6);
+            alternateSpanList.add(s4);
+
+            List<SpanNode> alternateChildren = new ArrayList<>();
+            Span s7 = new Span(1, 4);
+            Span s8 = new Span(1, 9);
+            Span s9 = new Span(5, 10);
+            alternateChildren.add(s7);
+            alternateChildren.add(s8);
+            alternateChildren.add(s9);
+
+            alternateSpanList.addChildren(alternateChildren, 5.55);
+
+            serializeAndAssert(alternateSpanList);
+        }
+    }
+
+    private void serializeAndAssert(AlternateSpanList alternateSpanList) {
+        GrowableByteBuffer buffer;
+        {
+            buffer = new GrowableByteBuffer(1024);
+            DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+            StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lk");
+            SpanTree tree = new SpanTree("bababa", alternateSpanList);
+            value.setSpanTree(tree);
+            serializer.write(null, value);
+            buffer.flip();
+        }
+        AlternateSpanList alternateSpanList2;
+        {
+            DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+            StringFieldValue value = new StringFieldValue();
+            deserializer.read(null, value);
+            alternateSpanList2 = (AlternateSpanList)value.getSpanTree("bababa").getRoot();
+        }
+
+        assertEquals(alternateSpanList, alternateSpanList2);
+        assertNotSame(alternateSpanList, alternateSpanList2);
+    }
+
+    @Test
+    public void testToString() {
+        SpanList root = new SpanList();
+        AlternateSpanList branch = new AlternateSpanList();
+
+        SpanNode sn1 = new Span(0, 5);
+        SpanNode span1 = new Span(0, 3);
+        root.add(sn1);
+        branch.add(span1);
+        root.add(branch);
+
+        SpanNode span11 = new Span(0, 3);
+        SpanNode span22 = new Span(1, 9);
+        SpanNode span33 = new Span(12, 10);
+
+        SpanList alternate = new SpanList();
+        alternate.add(span11);
+        alternate.add(span22);
+        alternate.add(span33);
+
+        List<SpanNode> subtreeList = new ArrayList<>();
+        subtreeList.add(alternate);
+
+        branch.setProbability(0, 100.0d);
+        branch.addChildren(subtreeList, 50.0d);
+
+        assertNotNull(root.toString());
+        assertNotNull(branch.toString());
+    }
+
+    @Test
+    public void testSortRecursive() {
+        AlternateSpanList root = new AlternateSpanList();
+        Span a1 = new Span(0, 1);
+        SpanList b1 = new SpanList();
+        SpanList c1 = new SpanList();
+        Span d1 = new Span(9, 1);
+        root.add(d1).add(c1).add(a1).add(b1);
+
+        Span aB2 = new Span(1, 1);
+        Span bB2 = new Span(2, 1);
+        Span cB2 = new Span(3, 1);
+        Span dB2 = new Span(4, 1);
+        b1.add(dB2).add(cB2).add(bB2).add(aB2);
+
+        Span aC2 = new Span(5, 1);
+        Span bC2 = new Span(6, 1);
+        Span cC2 = new Span(7, 1);
+        Span dC2 = new Span(8, 1);
+        c1.add(cC2).add(aC2).add(bC2).add(dC2);
+
+        Span altA1 = new Span(0, 1);
+        Span altB1 = new Span(1, 1);
+        List<SpanNode> altList = new ArrayList<>(2);
+        altList.add(altB1);
+        altList.add(altA1);
+        root.addChildren(1, altList, 0.5);
+
+        root.sortChildrenRecursive();
+        assertSame(a1, root.children().get(0));
+        assertSame(b1, root.children().get(1));
+        assertSame(c1, root.children().get(2));
+        assertSame(d1, root.children().get(3));
+        assertSame(aB2, b1.children().get(0));
+        assertSame(bB2, b1.children().get(1));
+        assertSame(cB2, b1.children().get(2));
+        assertSame(dB2, b1.children().get(3));
+        assertSame(aC2, c1.children().get(0));
+        assertSame(bC2, c1.children().get(1));
+        assertSame(cC2, c1.children().get(2));
+        assertSame(dC2, c1.children().get(3));
+
+        assertSame(altA1, root.children(1).get(0));
+        assertSame(altB1, root.children(1).get(1));
+    }
+
+    @Test
+    public void testIterator() {
+        AlternateSpanList asl1 = new AlternateSpanList();
+
+        Span span10 = new Span(1, 2);
+        Span span20 = new Span(2, 3);
+        Span span30 = new Span(3, 4);
+        List<SpanNode> subTree0 = new ArrayList<>(3);
+        subTree0.add(span10);
+        subTree0.add(span20);
+        subTree0.add(span30);
+
+        Span span11 = new Span(1, 2);
+        Span span21 = new Span(2, 3);
+        Span span31 = new Span(3, 4);
+        List<SpanNode> subTree1 = new ArrayList<>(3);
+        subTree1.add(span11);
+        subTree1.add(span21);
+        subTree1.add(span31);
+
+        Span span12 = new Span(1, 2);
+        Span span22 = new Span(2, 3);
+        Span span32 = new Span(3, 4);
+        List<SpanNode> subTree2 = new ArrayList<>(3);
+        subTree2.add(span12);
+        subTree2.add(span22);
+        subTree2.add(span32);
+
+        asl1.addChildren(0, subTree0, 0.1);
+        asl1.addChildren(1, subTree1, 0.1);
+        asl1.addChildren(2, subTree2, 0.1);
+
+        ListIterator<SpanNode> it = asl1.childIterator();
+        assertSame(span10, it.next());
+        assertSame(span20, it.next());
+        assertSame(span30, it.next());
+        assertSame(span11, it.next());
+        assertSame(span21, it.next());
+        assertSame(span31, it.next());
+        assertSame(span12, it.next());
+        assertSame(span22, it.next());
+        assertSame(span32, it.next());
+        assertFalse(it.hasNext());
+    }
+
+    @Test
+    public void testIteratorRecursive() {
+        AlternateSpanList asl1 = new AlternateSpanList();
+
+        Span span10 = new Span(1, 1);
+        Span span20 = new Span(2, 1);
+        Span span30 = new Span(3, 1);
+        List<SpanNode> subTree0 = new ArrayList<>(3);
+        subTree0.add(span10);
+        subTree0.add(span20);
+        subTree0.add(span30);
+
+        Span span11 = new Span(4, 1);
+        Span span21 = new Span(5, 1);
+        Span span31 = new Span(6, 1);
+        SpanList sl112 = new SpanList();
+        sl112.add(span11);
+        sl112.add(span21);
+        sl112.add(span31);
+        SpanList sl11 = new SpanList();
+        sl11.add(sl112);
+        List<SpanNode> subTree1 = new ArrayList<>(1);
+        subTree1.add(sl11);
+
+        Span span12 = new Span(7, 1);
+        Span span22 = new Span(8, 1);
+        Span span32 = new Span(9, 1);
+        List<SpanNode> subTree2 = new ArrayList<>(3);
+        subTree2.add(span12);
+        subTree2.add(span22);
+        subTree2.add(span32);
+
+        asl1.addChildren(0, subTree0, 0.1);
+        asl1.addChildren(1, subTree1, 0.1);
+        asl1.addChildren(2, subTree2, 0.1);
+
+        ListIterator<SpanNode> it = asl1.childIteratorRecursive();
+        assertSame(span10, it.next());
+        assertSame(span20, it.next());
+        assertSame(span30, it.next());
+        assertSame(span11, it.next());
+        assertSame(span21, it.next());
+        assertSame(span31, it.next());
+        assertSame(sl112, it.next());
+        assertSame(sl11, it.next());
+        assertSame(span12, it.next());
+        assertSame(span22, it.next());
+        assertSame(span32, it.next());
+        assertFalse(it.hasNext());
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/AnnotationTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AnnotationTestCase.java
new file mode 100644
index 00000000000..65ae706ec9a
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/AnnotationTestCase.java
@@ -0,0 +1,129 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AnnotationTestCase extends AbstractTypesTest {
+
+    @Test
+    public void testBasic() {
+        AnnotationType begTagType = new AnnotationType("begin_tag");
+        Annotation a = new Annotation(begTagType);
+        Annotation b = new Annotation(begTagType);
+
+        assertEquals(a, b);
+        assertEquals(a.hashCode(), b.hashCode());
+        assertNotSame(a, b);
+
+        Annotation c = new Annotation(new AnnotationType("determiner"));
+
+        assertFalse(a.equals(c));
+        assertFalse(c.equals(a));
+        assertFalse(b.equals(c));
+        assertFalse(c.equals(b));
+
+        assertFalse(a.hashCode() == c.hashCode());
+        assertFalse(c.hashCode() == a.hashCode());
+        assertFalse(b.hashCode() == c.hashCode());
+        assertFalse(c.hashCode() == b.hashCode());
+    }
+
+    @Test
+    public void testFieldValues() {
+        AnnotationType atype = new AnnotationType("foobar", DataType.STRING);
+        StringFieldValue sfv = new StringFieldValue("balloo");
+
+        Annotation a = new Annotation(atype);
+        a.setFieldValue(sfv);
+    }
+
+    @Test
+    public void testSerializeDeserialize() {
+        {
+            Annotation annotation = new Annotation(dummy);
+            serializeAndAssert(annotation);
+        }
+        {
+            Annotation annotation = new Annotation(number, new IntegerFieldValue(56));
+            serializeAndAssert(annotation);
+        }
+        {
+            Struct value = new Struct(person);
+            value.setFieldValue("firstname", "Barack");
+            value.setFieldValue("lastname", "Obama");
+            value.setFieldValue("birthyear", 1909);
+            Annotation annotation = new Annotation(personA, value);
+            serializeAndAssert(annotation);
+        }
+    }
+
+    /**
+     * A test case taken from real use to verify the API ease of use
+     */
+    @Test
+    public void testApi() {
+        // Prepare
+        AnnotationType type1 = new AnnotationType("type1", DataType.STRING);
+        AnnotationType type2 = new AnnotationType("type2", DataType.INT);
+        StringFieldValue output = new StringFieldValue("foo bar");
+        SpanTree tree;
+
+        // no shortcuts
+        {
+            SpanList root = new SpanList();
+            tree = new SpanTree("SpanTree1", root);
+            SpanNode node = new Span(0, 3);
+            tree.annotate(node, new Annotation(type1, new StringFieldValue("text")));
+            tree.annotate(node, new Annotation(type2, new IntegerFieldValue(1)));
+            root.add(node);
+            output.setSpanTree(tree);
+        }
+
+        // short
+        {
+            SpanList root = new SpanList();
+            output.setSpanTree(new SpanTree("SpanTree2", root));
+            SpanNode node = root.add(new Span(0, 3));
+            node.annotate(type1, "text").annotate(type2, 1);
+        }
+
+        // shorter
+        {
+            SpanList root = output.setSpanTree(new SpanTree("SpanTree3")).spanList();
+            root.span(0, 3).annotate(type1, "text").annotate(type2, 1);
+        }
+
+        // shortest
+        {
+            output.setSpanTree(new SpanTree("SpanTree4")).spanList().span(0, 3).annotate(type1, "text")
+                  .annotate(type2, 1);
+        }
+    }
+
+    private void serializeAndAssert(Annotation annotation) {
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+        serializer.write(annotation);
+        buffer.flip();
+
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+        Annotation annotation2 = new Annotation();
+        deserializer.read(annotation2);
+
+        assertEquals(annotation, annotation2);
+        assertNotSame(annotation, annotation2);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/AnnotationTypeRegistryTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AnnotationTypeRegistryTestCase.java
new file mode 100644
index 00000000000..273d30b5426
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/AnnotationTypeRegistryTestCase.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AnnotationTypeRegistryTestCase extends junit.framework.TestCase {
+    public void testRegisterUnregister() {
+        AnnotationTypeRegistry reg = new AnnotationTypeRegistry();
+        assertEquals(0, reg.getTypes().size());
+
+        AnnotationType one = new AnnotationType("one");
+        AnnotationType another = new AnnotationType("one");
+
+        //should work; re-registering type with same name and same id:
+        reg.register(one);
+        assertEquals(1, reg.getTypes().size());
+        reg.register(another);
+        assertEquals(1, reg.getTypes().size());
+
+        AnnotationType oneWithData = new AnnotationType("one", DataType.INT);
+
+        reg.register(oneWithData);
+        assertEquals(1, reg.getTypes().size());
+
+
+        AnnotationType two = new AnnotationType("two");
+        AnnotationType three = new AnnotationType("three");
+
+        reg.register(two);
+        assertEquals(2, reg.getTypes().size());
+        reg.register(three);
+        assertEquals(3, reg.getTypes().size());
+
+
+        reg.unregister("one");
+        assertEquals(2, reg.getTypes().size());
+        assertEquals("two", reg.getType("two").getName());
+        assertEquals("three", reg.getType("three").getName());
+
+        reg.unregister(two.getId());
+        assertEquals(1, reg.getTypes().size());
+        assertEquals("three", reg.getType("three").getName());
+
+        reg.unregister(three);
+        assertEquals(0, reg.getTypes().size());
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/AnnotationTypeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AnnotationTypeTestCase.java
new file mode 100644
index 00000000000..dcceb403564
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/AnnotationTypeTestCase.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AnnotationTypeTestCase extends junit.framework.TestCase {
+    public void testBasic() {
+        AnnotationType a = new AnnotationType("foo");
+        AnnotationType b = new AnnotationType("foo");
+
+        assertEquals(a, b);
+        assertEquals(a.hashCode(), b.hashCode());
+        assertEquals(a.hashCode(), a.getId());
+        assertEquals(b.hashCode(), b.getId());
+
+        AnnotationType c = new AnnotationType("bar");
+        assertEquals(c.hashCode(), c.getId());
+
+        assertFalse(a.equals(c));
+        assertFalse(c.equals(a));
+        assertFalse(b.equals(c));
+        assertFalse(c.equals(b));
+
+        assertFalse(a.hashCode() == c.hashCode());
+        assertFalse(c.hashCode() == a.hashCode());
+        assertFalse(b.hashCode() == c.hashCode());
+        assertFalse(c.hashCode() == b.hashCode());
+    }
+
+    public void testBasic2() {
+        AnnotationType a = new AnnotationType("foo", DataType.INT);
+        AnnotationType b = new AnnotationType("foo", DataType.INT);
+
+        assertEquals(a, b);
+        assertEquals(a.hashCode(), b.hashCode());
+        assertEquals(a.hashCode(), a.getId());
+        assertEquals(b.hashCode(), b.getId());
+
+        AnnotationType c = new AnnotationType("foo", DataType.FLOAT);
+        assertEquals(c.hashCode(), c.getId());
+
+        assertEquals(a, c);
+        assertEquals(a.hashCode(), c.hashCode());
+        assertEquals(a.hashCode(), a.getId());
+        assertEquals(c.hashCode(), c.getId());
+    }
+
+    public void testPolymorphy() {
+        AnnotationType suuper = new AnnotationType("super");
+        AnnotationType sub = new AnnotationType("sub");
+        sub.inherit(suuper);
+
+        //reference type for super annotation type
+        AnnotationReferenceDataType refType = new AnnotationReferenceDataType(suuper);
+
+        Annotation superAnnotation = new Annotation(suuper);
+        Annotation subAnnotation = new Annotation(sub);
+
+        AnnotationReference ref1 = new AnnotationReference(refType, superAnnotation);
+        //this would fail without polymorphy support:
+        AnnotationReference ref2 = new AnnotationReference(refType, subAnnotation);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/AnnotationTypesTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AnnotationTypesTestCase.java
new file mode 100644
index 00000000000..7a2d3d006b2
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/AnnotationTypesTestCase.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class AnnotationTypesTestCase {
+
+    @Test
+    public void requireThatProximityBreakAcceptsDoubleWeight() {
+        try {
+            new Annotation(AnnotationTypes.PROXIMITY_BREAK, new DoubleFieldValue(6.9));
+        } catch (Exception e) {
+            fail("this is required for ticket #665166, do not change");
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug4155865TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug4155865TestCase.java
new file mode 100644
index 00000000000..6cf9ee8f683
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/Bug4155865TestCase.java
@@ -0,0 +1,379 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class Bug4155865TestCase {
+
+    private SpanTree tree;
+
+    @Before
+    public void buildTree() {
+        AnnotationType at1 = new AnnotationType("person", DataType.STRING);
+        AnnotationType at2 = new AnnotationType("street", DataType.STRING);
+        AnnotationType at3 = new AnnotationType("city", DataType.STRING);
+
+        StringFieldValue fv1 = (StringFieldValue) at1.getDataType().createFieldValue();
+        fv1.assign("Praveen");
+        Annotation an1 = new Annotation(at1, fv1);
+
+        StringFieldValue fv2 = (StringFieldValue) at2.getDataType().createFieldValue();
+        fv2.assign("Bommanahalli");
+        Annotation an2 = new Annotation(at2, fv2);
+
+        StringFieldValue fv3 = (StringFieldValue) at3.getDataType().createFieldValue();
+        fv3.assign("Bangalore");
+        Annotation an3 = new Annotation(at3, fv3);
+
+
+        StringFieldValue fv11 = (StringFieldValue) at1.getDataType().createFieldValue();
+        fv11.assign("Elancheran");
+        Annotation an11 = new Annotation(at1, fv11);
+
+        StringFieldValue fv22 = (StringFieldValue) at2.getDataType().createFieldValue();
+        fv22.assign("Kagadaspura");
+        Annotation an22 = new Annotation(at2, fv22);
+
+        StringFieldValue fv33 = (StringFieldValue) at3.getDataType().createFieldValue();
+        fv33.assign("Delhi");
+        Annotation an33 = new Annotation(at3, fv33);
+
+        StringFieldValue fv111 = (StringFieldValue) at1.getDataType().createFieldValue();
+        fv111.assign("Govindan");
+        Annotation an111 = new Annotation(at1, fv111);
+
+        StringFieldValue fv222 = (StringFieldValue) at1.getDataType().createFieldValue();
+        fv222.assign("Kenneth");
+        Annotation an222 = new Annotation(at2, fv222);
+
+        SpanList root = new SpanList();
+        tree = new SpanTree("test", root);
+        AlternateSpanList branch = new AlternateSpanList();
+        SpanList branch2 = new SpanList();
+
+
+        SpanNode sn1 = new Span(0, 5);
+        SpanNode sn2 = new Span(5, 10);
+        SpanNode sn3 = new Span(15, 10);
+        SpanNode sn4 = new Span(15, 20);
+
+        SpanNode span1 = new Span(0, 3);
+        SpanNode span2 = new Span(1, 9);
+        SpanNode span3 = new Span(12, 10);
+
+        root.add(sn4);
+        root.add(sn3);
+        root.add(sn2);
+        root.add(sn1);
+
+        SpanNode spn1 = new Span(4, 5);
+        branch2.add(spn1);
+
+        AlternateSpanList branch3 = new AlternateSpanList();
+
+        SpanNode spn2 = new Span(1, 4);
+        SpanNode spn3 = new Span(6, 10);
+        tree.annotate(spn2, an111);
+        tree.annotate(spn3, an222);
+
+        List<SpanNode> stList = new ArrayList<SpanNode>();
+        stList.add(spn2);
+        List<SpanNode> stList1 = new ArrayList<SpanNode>();
+        stList1.add(spn3);
+        branch3.addChildren(stList, 45.0);
+        branch3.addChildren(stList1, 25.0);
+
+        root.add(branch2);
+        branch.add(branch3);
+        root.add(branch);
+
+        SpanNode span11 = new Span(0, 3);
+        SpanNode span22 = new Span(1, 9);
+        SpanNode span33 = new Span(12, 10);
+
+        SpanList alternate2 = new SpanList();
+        alternate2.add(span3);
+        alternate2.add(span2);
+        alternate2.add(span1);
+
+        SpanList alternate1 = new SpanList();
+        alternate1.add(span11);
+        alternate1.add(span22);
+        alternate1.add(span33);
+
+        tree.annotate(span1, an1);
+        tree.annotate(span2, an2);
+        tree.annotate(span3, an3);
+
+        tree.annotate(span11, an11);
+        tree.annotate(span22, an22);
+        tree.annotate(span33, an33);
+
+        List<SpanNode> subtreeList1 = new ArrayList<SpanNode>();
+        subtreeList1.add(alternate1);
+
+        List<SpanNode> subtreeList2 = new ArrayList<SpanNode>();
+        subtreeList2.add(alternate2);
+
+        branch.addChildren(subtreeList1, 50.0d);
+        branch.addChildren(subtreeList2, 100.0d);
+    }
+
+    @Test
+    public void assertTree() {
+        final SpanList root = (SpanList) tree.getRoot();
+        //Level 0:
+        assertEquals(0, root.getFrom());
+        assertEquals(35, root.getTo());
+        assertEquals(6, root.numChildren());
+
+        //Level 1:
+        assertTrue(root.children().get(0) instanceof Span);
+        assertEquals(15, root.children().get(0).getFrom());
+        assertEquals(35, root.children().get(0).getTo());
+        assertFalse(root.children().get(0).childIterator().hasNext());
+        assertFalse(tree.iterator(root.children().get(0)).hasNext());
+
+        assertTrue(root.children().get(1) instanceof Span);
+        assertEquals(15, root.children().get(1).getFrom());
+        assertEquals(25, root.children().get(1).getTo());
+        assertFalse(root.children().get(1).childIterator().hasNext());
+        assertFalse(tree.iterator(root.children().get(1)).hasNext());
+
+        assertTrue(root.children().get(2) instanceof Span);
+        assertEquals(5, root.children().get(2).getFrom());
+        assertEquals(15, root.children().get(2).getTo());
+        assertFalse(root.children().get(2).childIterator().hasNext());
+        assertFalse(tree.iterator(root.children().get(2)).hasNext());
+
+        assertTrue(root.children().get(3) instanceof Span);
+        assertEquals(0, root.children().get(3).getFrom());
+        assertEquals(5, root.children().get(3).getTo());
+        assertFalse(root.children().get(3).childIterator().hasNext());
+        assertFalse(tree.iterator(root.children().get(3)).hasNext());
+
+        assertTrue(root.children().get(4) instanceof SpanList);
+        assertFalse(root.children().get(4) instanceof AlternateSpanList);
+        assertEquals(4, root.children().get(4).getFrom());
+        assertEquals(9, root.children().get(4).getTo());
+        assertTrue(root.children().get(4).childIterator().hasNext());
+        assertFalse(tree.iterator(root.children().get(4)).hasNext());
+
+        assertTrue(root.children().get(5) instanceof AlternateSpanList);
+        assertEquals(-1, root.children().get(5).getFrom());
+        assertEquals(-1, root.children().get(5).getTo());
+        assertTrue(root.children().get(5).childIterator().hasNext());
+        assertFalse(tree.iterator(root.children().get(5)).hasNext());
+
+
+        //Level 2:
+        final SpanList list1 = (SpanList) root.children().get(4);
+        assertFalse(list1 instanceof AlternateSpanList);
+        assertEquals(1, list1.numChildren());
+
+        assertTrue(list1.children().get(0) instanceof Span);
+        assertEquals(4, list1.children().get(0).getFrom());
+        assertEquals(9, list1.children().get(0).getTo());
+        assertFalse(list1.children().get(0).childIterator().hasNext());
+        assertFalse(tree.iterator(list1.children().get(0)).hasNext());
+
+        final AlternateSpanList altList1 = (AlternateSpanList) root.children().get(5);
+        assertEquals(3, altList1.getNumSubTrees());
+
+        List<SpanNode> subTree0 = altList1.children(0);
+        assertEquals(1, subTree0.size());
+
+        assertTrue(subTree0.get(0) instanceof AlternateSpanList); //TODO: Assert on this!!
+        assertEquals(3, ((AlternateSpanList) subTree0.get(0)).getNumSubTrees());
+        List<SpanNode> subTree0_0 = ((AlternateSpanList) subTree0.get(0)).children(0);
+        assertEquals(0, subTree0_0.size());
+        List<SpanNode> subTree0_1 = ((AlternateSpanList) subTree0.get(0)).children(1);
+        assertEquals(1, subTree0_1.size());
+        List<SpanNode> subTree0_2 = ((AlternateSpanList) subTree0.get(0)).children(2);
+        assertEquals(1, subTree0_2.size());
+
+        assertEquals(-1, subTree0.get(0).getFrom());
+        assertEquals(-1, subTree0.get(0).getTo());
+        assertTrue(subTree0.get(0).childIterator().hasNext());
+        assertFalse(tree.iterator(subTree0.get(0)).hasNext());
+
+        List<SpanNode> subTree1 = altList1.children(1);
+        assertEquals(1, subTree1.size());
+
+        assertTrue(subTree1.get(0) instanceof SpanList);
+        assertFalse(subTree1.get(0) instanceof AlternateSpanList);
+        assertEquals(0, subTree1.get(0).getFrom());
+        assertEquals(22, subTree1.get(0).getTo());
+        assertTrue(subTree1.get(0).childIterator().hasNext());
+        assertFalse(tree.iterator(subTree1.get(0)).hasNext());
+
+        List<SpanNode> subTree2 = altList1.children(2);
+        assertEquals(1, subTree2.size());
+
+        assertTrue(subTree2.get(0) instanceof SpanList);
+        assertFalse(subTree2.get(0) instanceof AlternateSpanList);
+        assertEquals(0, subTree2.get(0).getFrom());
+        assertEquals(22, subTree2.get(0).getTo());
+        assertTrue(subTree2.get(0).childIterator().hasNext());
+        assertFalse(tree.iterator(subTree2.get(0)).hasNext());
+
+        //NOTE subTree2 has children
+
+
+        //Level 3
+        assertTrue(subTree0_0.isEmpty());
+
+        final Span subTree0_1_0 = (Span) subTree0_1.get(0);
+        assertEquals(1, subTree0_1_0.getFrom());
+        assertEquals(5, subTree0_1_0.getTo());
+        assertFalse(subTree0_1_0.childIterator().hasNext());
+        final Iterator<Annotation> subTree0_1_0AnnIterator = tree.iterator(subTree0_1_0);
+        assertTrue(subTree0_1_0AnnIterator.hasNext());
+        Annotation subTree0_1_0Annotation = subTree0_1_0AnnIterator.next();
+        //TODO: Assert on annotation
+        assertFalse(subTree0_1_0AnnIterator.hasNext());
+
+
+        final Span subTree0_2_0 = (Span) subTree0_2.get(0);
+        assertEquals(6, subTree0_2_0.getFrom());
+        assertEquals(16, subTree0_2_0.getTo());
+        assertFalse(subTree0_2_0.childIterator().hasNext());
+        final Iterator<Annotation> subTree0_2_0AnnIterator = tree.iterator(subTree0_2_0);
+        assertTrue(subTree0_2_0AnnIterator.hasNext());
+        Annotation subTree0_2_0Annotation = subTree0_2_0AnnIterator.next();
+        //TODO: Assert on annotation
+        assertFalse(subTree0_2_0AnnIterator.hasNext());
+
+
+        final SpanList sl = (SpanList) subTree1.get(0);
+        assertFalse(sl instanceof AlternateSpanList);
+        assertEquals(3, sl.children().size());
+
+        assertTrue(sl.children().get(0) instanceof Span);
+        assertEquals(0, sl.children().get(0).getFrom());
+        assertEquals(3, sl.children().get(0).getTo());
+        assertFalse(sl.children().get(0).childIterator().hasNext());
+        final Iterator<Annotation> iterator0 = tree.iterator(sl.children().get(0));
+        assertTrue(iterator0.hasNext());
+        Annotation iterator0Annotation = iterator0.next();
+        //TODO: Assert on annotation
+        assertFalse(iterator0.hasNext());
+
+        assertTrue(sl.children().get(1) instanceof Span);
+        assertEquals(1, sl.children().get(1).getFrom());
+        assertEquals(10, sl.children().get(1).getTo());
+        assertFalse(sl.children().get(1).childIterator().hasNext());
+        final Iterator<Annotation> iterator1 = tree.iterator(sl.children().get(1));
+        assertTrue(iterator1.hasNext());
+        Annotation iterator1Annotation = iterator1.next();
+        //TODO: Assert on annotation
+        assertFalse(iterator1.hasNext());
+
+        assertTrue(sl.children().get(2) instanceof Span);
+        assertEquals(12, sl.children().get(2).getFrom());
+        assertEquals(22, sl.children().get(2).getTo());
+        assertFalse(sl.children().get(2).childIterator().hasNext());
+        final Iterator<Annotation> iterator2 = tree.iterator(sl.children().get(2));
+        assertTrue(iterator2.hasNext());
+        Annotation iterator2Annotation = iterator2.next();
+        //TODO: Assert on annotation
+        assertFalse(iterator2.hasNext());
+
+        final SpanList sl2 = (SpanList) subTree2.get(0);
+        assertFalse (sl2 instanceof AlternateSpanList);
+        assertEquals(3, sl2.children().size());
+
+        assertTrue(sl2.children().get(0) instanceof Span);
+        assertEquals(12, sl2.children().get(0).getFrom());
+        assertEquals(22, sl2.children().get(0).getTo());
+        assertFalse(sl2.children().get(0).childIterator().hasNext());
+        final Iterator<Annotation> iterator3 = tree.iterator(sl2.children().get(0));
+        assertTrue(iterator3.hasNext());
+        Annotation iterator3Annotation = iterator3.next();
+        //TODO: Assert on annotation
+        assertFalse(iterator3.hasNext());
+
+        assertTrue(sl2.children().get(1) instanceof Span);
+        assertEquals(1, sl2.children().get(1).getFrom());
+        assertEquals(10, sl2.children().get(1).getTo());
+        assertFalse(sl2.children().get(1).childIterator().hasNext());
+        final Iterator<Annotation> iterator4 = tree.iterator(sl2.children().get(1));
+        assertTrue(iterator4.hasNext());
+        Annotation iterator4Annotation = iterator4.next();
+        //TODO: Assert on annotation
+        assertFalse(iterator4.hasNext());
+
+        assertTrue(sl2.children().get(2) instanceof Span);
+        assertEquals(0, sl2.children().get(2).getFrom());
+        assertEquals(3, sl2.children().get(2).getTo());
+        assertFalse(sl2.children().get(2).childIterator().hasNext());
+        final Iterator<Annotation> iterator5 = tree.iterator(sl2.children().get(2));
+        assertTrue(iterator5.hasNext());
+        Annotation iterator5Annotation = iterator5.next();
+        //TODO: Assert on annotation
+        assertFalse(iterator5.hasNext());
+
+    }
+
+    @After
+    public void removeTree() {
+        tree = null;
+    }
+
+    public void parseAlternateLists(SpanTree tree, AlternateSpanList aspl) {
+        int no = aspl.getNumSubTrees();
+        System.out.println("Parsing Alternate span list. No of subtrees: " + no);
+        int ctr = 0;
+        while (ctr < no) {
+            System.out.println("\nSubTree: " + ctr);
+            ListIterator<SpanNode> lIter = aspl.childIteratorRecursive(ctr);
+            while (lIter.hasNext()) {
+                SpanNode spnNode = lIter.next();
+                System.out.println("Parsing span node: [" + spnNode.getFrom() + ", " + spnNode.getTo() + "] ");
+                if (spnNode instanceof AlternateSpanList) {
+                    System.out.println("A child alternate span list found. Recursing");
+                    parseAlternateLists(tree, (AlternateSpanList)spnNode);
+                }
+
+                getAnnotationsForNode(tree, spnNode);
+            }
+            ctr ++;
+        }
+    }
+
+    public void getAnnotationsForNode(SpanTree tree, SpanNode node) {
+        Iterator<Annotation> iter = tree.iterator(node);
+        boolean annotationPresent = false;
+        while (iter.hasNext()) {
+            annotationPresent = true;
+            Annotation xx = iter.next();
+            StringFieldValue fValue = (StringFieldValue) xx.getFieldValue();
+            System.out.println("Annotation: " + xx);
+            if (fValue == null) {
+                System.out.println("Field Value is null");
+                return;
+            } else {
+                System.out.println("Field Value: " + fValue.getString());
+            }
+        }
+        if (!annotationPresent) {
+            System.out.println("****No annotations found for the span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug4164299TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug4164299TestCase.java
new file mode 100644
index 00000000000..3a88fb98f48
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/Bug4164299TestCase.java
@@ -0,0 +1,140 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:mpraveen@yahoo-inc.com">Praveen Mohan</a>
+ *
+ * This test checks if sub-trees are sorted appropriately by probability
+ * within an alternate span list and getFrom() and getTo() are updated properly
+ * when the trees are sorted.
+ *
+ */
+public class Bug4164299TestCase {
+
+    private SpanTree tree;
+
+    @Before
+    public void buildTree() {
+        SpanList root = new SpanList();
+        AlternateSpanList branch = new AlternateSpanList();
+        tree = new SpanTree("test", root);
+
+        SpanNode span1 = new Span(0, 2);
+        SpanNode span2 = new Span(2, 2);
+        SpanNode span3 = new Span(4, 2);
+
+        SpanNode span11 = new Span(10, 2);
+        SpanNode span22 = new Span(12, 2);
+        SpanNode span33 = new Span(14, 2);
+
+        branch.add(span3);
+        branch.add(span2);
+        branch.add(span1);
+
+        List<SpanNode> subtreeList = new ArrayList<SpanNode>();
+        subtreeList.add(span11);
+        subtreeList.add(span22);
+        subtreeList.add(span33);
+        branch.addChildren(1, subtreeList, 50.0d);
+        root.add(branch);
+
+    }
+
+    @Test
+    public void assertTree() {
+        final AlternateSpanList branch =  (AlternateSpanList)((SpanList)tree.getRoot()).children().get(0);
+        assertEquals(0, branch.getFrom());
+        assertEquals(6, branch.getTo());
+        assertEquals(10, branch.getFrom(1));
+        assertEquals(16, branch.getTo(1));
+        branch.sortSubTreesByProbability();
+
+        assertEquals(10, branch.getFrom());
+        assertEquals(16, branch.getTo());
+        assertEquals(0, branch.getFrom(1));
+        assertEquals(6, branch.getTo(1));
+    }
+
+    @After
+    public void removeTree() {
+        tree = null;
+    }
+
+    public void consumeAnnotations(SpanTree tree, SpanList root) {
+        System.out.println("\n\nSpanList: [" + root.getFrom() + ", " + root.getTo() + "] num Children: " + root.numChildren());
+        System.out.println("-------------------");
+        Iterator<SpanNode> childIterator = root.childIterator();
+        while (childIterator.hasNext()) {
+            SpanNode node = childIterator.next();
+            System.out.println("\n\nSpan Node (" + node + "): [" + node.getFrom() + ", " + node.getTo() + "] ");
+            if (node instanceof AlternateSpanList) {
+                parseAlternateLists(tree, (AlternateSpanList)node);
+                System.out.println("---- Alternate SpanList complete ---");
+            } else if (node instanceof SpanList) {
+                System.out.println("Encountered another span list");
+                SpanList spl = (SpanList) node;
+                ListIterator<SpanNode> lli = spl.childIterator();
+                while (lli.hasNext()) System.out.print(" " + lli.next() + " ");
+                consumeAnnotations(tree, (SpanList) node);
+            } else {
+                   System.out.println("\nGetting annotations for this span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+                   getAnnotationsForNode(tree, node);
+            }
+        }
+        System.out.println("\nGetting annotations for the SpanList itself : [" + root.getFrom() + ", " + root.getTo() + "] ");
+        getAnnotationsForNode(tree, root);
+    }
+
+    public void parseAlternateLists(SpanTree tree, AlternateSpanList aspl) {
+        int no = aspl.getNumSubTrees();
+        System.out.println("Parsing Alternate span list. No of subtrees: " + no);
+        int ctr = 0;
+        while (ctr < no) {
+            System.out.println("\nSubTree: " + ctr);
+            ListIterator<SpanNode> lIter = aspl.childIteratorRecursive(ctr);
+            while (lIter.hasNext()) {
+                SpanNode spnNode = lIter.next();
+                System.out.println("Parsing span node: [" + spnNode.getFrom() + ", " + spnNode.getTo() + "] ");
+                if (spnNode instanceof AlternateSpanList) {
+                    System.out.println("A child alternate span list found. Recursing");
+                    parseAlternateLists(tree, (AlternateSpanList)spnNode);
+                }
+
+                getAnnotationsForNode(tree, spnNode);
+            }
+            ctr ++;
+        }
+    }
+
+    public void getAnnotationsForNode(SpanTree tree, SpanNode node) {
+        Iterator<Annotation> iter = tree.iterator(node);
+        boolean annotationPresent = false;
+        while (iter.hasNext()) {
+            annotationPresent = true;
+            Annotation xx = iter.next();
+            StringFieldValue fValue = (StringFieldValue) xx.getFieldValue();
+            System.out.println("Annotation: " + xx);
+            if (fValue == null) {
+                System.out.println("Field Value is null");
+                return;
+            } else {
+                System.out.println("Field Value: " + fValue.getString());
+            }
+        }
+        if (!annotationPresent) {
+            System.out.println("****No annotations found for the span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug4259784TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug4259784TestCase.java
new file mode 100644
index 00000000000..035f6a95a19
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/Bug4259784TestCase.java
@@ -0,0 +1,129 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Bug4259784TestCase extends junit.framework.TestCase {
+
+    @Test
+    public void testSerialize() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/java/com/yahoo/document/annotation/documentmanager.bug4259784.cfg");
+
+        DocumentType type = manager.getDocumentType("blog");
+        Document doc = new Document(type, "doc:this:is:a:test");
+        doc.setFieldValue("body", new StringFieldValue("bla bla bla bla bla bla bla" +
+                                                       "bla bla bla bla bla bla bla"));
+        annotate(doc, manager);
+
+        GrowableByteBuffer buf = new GrowableByteBuffer();
+        doc.serialize(buf);
+    }
+
+
+    private void annotate(Document document, DocumentTypeManager manager) {
+        AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();
+
+		AnnotationType company = registry.getType("company");
+		AnnotationType industry = registry.getType("industry");
+		AnnotationType person  = registry.getType("person");
+		AnnotationType location = registry.getType("location");
+
+	    SpanTree tree = new SpanTree("testannotations");
+        SpanList root = (SpanList) tree.getRoot();
+
+	    SpanNode span1 = new Span(0,5);
+	    SpanNode span2 = new Span(5,10);
+	    SpanNode span3 = new Span(10,15);
+	    SpanNode span4 = new Span(15,20);
+	    SpanNode span5 = new Span(6,10);
+	    SpanNode span6 = new Span(8,4);
+        SpanNode span7 = new Span(4, 2);
+
+	    root.add(span1);
+	    root.add(span2);
+	    root.add(span4);
+        root.add(span5);
+        root.add(span6);
+
+        AlternateSpanList aspl = new AlternateSpanList();
+        aspl.add(span7);
+        List<SpanNode> subtree1 = new ArrayList<SpanNode>();
+        subtree1.add(span3);
+        aspl.addChildren(1, subtree1, 33.0d);
+
+        root.add(aspl);
+
+	    Struct personValue = (Struct) person.getDataType().createFieldValue();
+	    personValue.setFieldValue("name", "Richard Bair");
+	    Annotation personAn = new Annotation(person, personValue);
+	    tree.annotate(span1, personAn);
+
+		Struct companyValue = (Struct) company.getDataType().createFieldValue();
+		companyValue.setFieldValue("name", "Sun");
+        Annotation compAn = new Annotation(company, companyValue);
+        tree.annotate(span2, compAn);
+
+        Struct locationVal = new Struct(manager.getDataType("annotation.location"));
+		locationVal.setFieldValue("lat", 37.774929);
+		locationVal.setFieldValue("lon", -122.419415);
+        Annotation locAnnotation = new Annotation(location, locationVal);
+        tree.annotate(span3, locAnnotation);
+
+
+        Struct dirValue1 = new Struct(manager.getDataType("annotation.person"));
+        dirValue1.setFieldValue("name", "Jonathan Schwartz");
+        Annotation dirAnnotation1 = new Annotation(person, dirValue1);
+        tree.annotate(span5, dirAnnotation1);
+
+        Struct dirValue2 = new Struct(manager.getDataType("annotation.person"));
+        dirValue2.setFieldValue("name", "Scott Mcnealy");
+        Annotation dirAnnotation2 = new Annotation(person, dirValue2);
+        tree.annotate(span6, dirAnnotation2);
+
+
+        Struct indValue = new Struct(manager.getDataType("annotation.industry"));
+        indValue.setFieldValue("vertical", "Manufacturing");
+        Annotation indAn = new Annotation(industry, indValue);
+        tree.annotate(span4, indAn);
+
+
+        Field compLocField = ((StructDataType) company.getDataType()).getField("place");
+        AnnotationReferenceDataType annType = (AnnotationReferenceDataType) compLocField.getDataType();
+        FieldValue compLocFieldVal = new AnnotationReference(annType, locAnnotation);
+		companyValue.setFieldValue(compLocField, compLocFieldVal);
+		companyValue.setFieldValue("vertical", "software");
+
+
+
+		Field dirField = ((StructDataType) company.getDataType()).getField("directors");
+		Array<FieldValue> dirFieldVal = new Array<FieldValue>(dirField.getDataType());
+		AnnotationReferenceDataType annRefType = (AnnotationReferenceDataType) ((ArrayDataType) dirField.getDataType()).getNestedType();
+		dirFieldVal.add(new AnnotationReference(annRefType, dirAnnotation1));
+		dirFieldVal.add(new AnnotationReference(annRefType, dirAnnotation2));
+		companyValue.setFieldValue(dirField, dirFieldVal);
+
+        tree.clearAnnotations(span3);
+
+        StringFieldValue body = (StringFieldValue) document.getFieldValue(document.getDataType().getField("body"));
+		body.setSpanTree(tree);
+	    document.setFieldValue(document.getDataType().getField("body"), body);
+	}
+
+}
+
diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug4261985TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug4261985TestCase.java
new file mode 100644
index 00000000000..4d6d040bcf8
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/Bug4261985TestCase.java
@@ -0,0 +1,153 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Bug4261985TestCase {
+
+    @Test
+    public void testAnnotate() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/java/com/yahoo/document/annotation/documentmanager.bug4261985.cfg");
+
+        DocumentType type = manager.getDocumentType("blog");
+        Document doc = new Document(type, "doc:this:is:a:test");
+        doc.setFieldValue("body", new StringFieldValue("bla bla bla bla bla bla bla" +
+                                                       "bla bla bla bla bla bla bla"));
+        annotate(doc, manager);
+
+        GrowableByteBuffer buf = new GrowableByteBuffer();
+        doc.serialize(buf);
+    }
+
+    public void annotate(Document document, DocumentTypeManager manager) {
+        AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();
+
+		AnnotationType company = registry.getType("company");
+		AnnotationType industry = registry.getType("industry");
+		AnnotationType person  = registry.getType("person");
+		AnnotationType location = registry.getType("location");
+		AnnotationType bigshots = registry.getType("bigshots");
+
+		if (company.inherits(industry)) {
+			System.out.println("Company Inherits Industry");
+		} else {
+			System.out.println("FAIL: COMPANY DOES NOT INHERIT INDUSTRY");
+            throw new RuntimeException("FAIL: COMPANY DOES NOT INHERIT INDUSTRY, though it does in SD file");
+		}
+
+	    SpanTree tree = new SpanTree("testannotations");
+        SpanList root = (SpanList) tree.getRoot();
+
+	    SpanNode span1 = new Span(0,5);
+	    SpanNode span2 = new Span(5,10);
+	    SpanNode span3 = new Span(10,15);
+	    SpanNode span4 = new Span(15,20);
+	    SpanNode span5 = new Span(6,10);
+	    SpanNode span6 = new Span(8,4);
+        SpanNode span7 = new Span(4, 2);
+        SpanNode span8 = new Span(12, 6);
+
+	    root.add(span1);
+	    root.add(span2);
+	    //root.add(span3);
+	    root.add(span4);
+        root.add(span5);
+        root.add(span6);
+        //root.add(span7);
+        root.add(span8);
+
+        AlternateSpanList aspl = new AlternateSpanList();
+        aspl.add(span7);
+        List<SpanNode> subtree1 = new ArrayList<SpanNode>();
+        subtree1.add(span3);
+        aspl.addChildren(1, subtree1, 33.0d);
+
+        root.add(aspl);
+
+	    Struct personValue = (Struct) person.getDataType().createFieldValue();
+	    personValue.setFieldValue("name", "Richard Bair");
+	    Annotation personAn = new Annotation(person, personValue);
+	    tree.annotate(span1, personAn);
+
+		Struct companyValue = (Struct) company.getDataType().createFieldValue();
+		companyValue.setFieldValue("name", "Sun");
+
+        Struct locationVal = new Struct(manager.getDataType("annotation.location"));
+		locationVal.setFieldValue("lat", 37.774929);
+		locationVal.setFieldValue("lon", -122.419415);
+        Annotation locAnnotation = new Annotation(location, locationVal);
+        Field compLocField = ((StructDataType) company.getDataType()).getField("place");
+		//FieldValue compLocFieldVal = new FieldValue(compLocField.getDataType());
+        AnnotationReferenceDataType annType = (AnnotationReferenceDataType) compLocField.getDataType();
+        FieldValue compLocFieldVal = null;
+        //if (scenario.equals("createFieldValue")) {
+         //   compLocFieldVal = annType.createFieldValue(new AnnotationReference(annType, locAnnotation));
+        //} else {
+            compLocFieldVal = new AnnotationReference(annType, locAnnotation);
+        //}
+		companyValue.setFieldValue(compLocField, compLocFieldVal);
+
+		companyValue.setFieldValue("vertical", "software");
+		Struct dirValue1 = new Struct(manager.getDataType("annotation.person"));
+	    dirValue1.setFieldValue("name", "Jonathan Schwartz");
+	    Annotation dirAnnotation1 = new Annotation(person, dirValue1);
+	    Struct dirValue2 = new Struct(manager.getDataType("annotation.person"));
+	    dirValue2.setFieldValue("name", "Scott Mcnealy");
+	    Annotation dirAnnotation2 = new Annotation(person, dirValue2);
+		Field dirField = ((StructDataType) company.getDataType()).getField("directors");
+		Array<FieldValue> dirFieldVal = new Array<FieldValue>(dirField.getDataType());
+		AnnotationReferenceDataType annRefType = (AnnotationReferenceDataType) ((ArrayDataType) dirField.getDataType()).getNestedType();
+		dirFieldVal.add(new AnnotationReference(annRefType, dirAnnotation1));
+		dirFieldVal.add(new AnnotationReference(annRefType, dirAnnotation2));
+		companyValue.setFieldValue(dirField, dirFieldVal);
+		Annotation compAn = new Annotation(company, companyValue);
+		tree.annotate(span2, compAn);
+
+		Struct bigshotsValue = (Struct) bigshots.getDataType().createFieldValue();
+		Field ceosField = ((StructDataType) bigshots.getDataType()).getField("ceos");
+		//FieldValue compLocFieldVal = new FieldValue(compLocField.getDataType());
+        AnnotationReferenceDataType annType1 = (AnnotationReferenceDataType) ceosField.getDataType();
+        FieldValue ceosFieldVal = new AnnotationReference(annType1, compAn);
+		bigshotsValue.setFieldValue(ceosField, ceosFieldVal);
+
+		Annotation bigshotsAn = new Annotation(bigshots, bigshotsValue);
+		tree.annotate(span8, bigshotsAn);
+
+		Field selfField = ((StructDataType) bigshots.getDataType()).getField("self");
+		AnnotationReferenceDataType annType2 = (AnnotationReferenceDataType) selfField.getDataType();
+		FieldValue selfFieldVal = new AnnotationReference(annType2, bigshotsAn);
+		bigshotsValue.setFieldValue(selfField, selfFieldVal);
+		bigshotsAn = new Annotation(bigshots, bigshotsValue);
+		tree.annotate(span8, bigshotsAn);
+
+	    tree.annotate(span3, locAnnotation);
+        tree.annotate(span5, dirAnnotation1);
+        tree.annotate(span6, dirAnnotation2);
+
+		Struct indValue = new Struct(manager.getDataType("annotation.industry"));
+		indValue.setFieldValue("vertical", "Manufacturing");
+		Annotation indAn = new Annotation(industry, indValue);
+		tree.annotate(span4, indAn);
+
+        StringFieldValue body = (StringFieldValue) document.getFieldValue(document.getDataType().getField("body"));
+		body.setSpanTree(tree);
+	    document.setFieldValue(document.getDataType().getField("body"), body);
+	}
+}
+
diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug4475379TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug4475379TestCase.java
new file mode 100755
index 00000000000..c75e7300773
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/Bug4475379TestCase.java
@@ -0,0 +1,151 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class Bug4475379TestCase {
+
+    @Test
+    public void testClone() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/java/com/yahoo/document/annotation/documentmanager.bug4475379.cfg");
+
+        DocumentType type = manager.getDocumentType("blog");
+        Document doc = new Document(type, "doc:this:is:a:test");
+        doc.setFieldValue("body", new StringFieldValue(""));
+        annotate(manager, doc);
+
+        Document anotherDoc = doc.clone();
+        assertThat(doc, equalTo(anotherDoc));
+    }
+
+    public void annotate(DocumentTypeManager manager, Document document) {
+        AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();
+
+        AnnotationType company = registry.getType("company");
+        AnnotationType industry = registry.getType("industry");
+        AnnotationType person = registry.getType("person");
+        AnnotationType location = registry.getType("location");
+
+        Annotation compAn1;
+        Annotation personAn1;
+        Annotation locAn1;
+        Annotation indAn1;
+        Annotation compAn2;
+        Annotation personAn2;
+        Annotation locAn2;
+        Annotation indAn2;
+
+        {
+            Struct companyValue1 = (Struct) company.getDataType().createFieldValue();
+            companyValue1.setFieldValue("name", new StringFieldValue("Sun"));
+            companyValue1.setFieldValue("ceo", new StringFieldValue("Scott Mcnealy"));
+            companyValue1.setFieldValue("lat", new FloatFieldValue(37.7f));
+            companyValue1.setFieldValue("lon", new FloatFieldValue(-122.44f));
+            companyValue1.setFieldValue("alt", 60.456);
+            companyValue1.setFieldValue("vertical", new StringFieldValue("software"));
+            compAn1 = new Annotation(company, companyValue1);
+        }
+        {
+            Struct personValue1 = new Struct(manager.getDataType("annotation.person"));
+            personValue1.setFieldValue("name", new StringFieldValue("Richard Bair"));
+            personAn1 = new Annotation(person, personValue1);
+        }
+        {
+            Struct locValue1 = new Struct(manager.getDataType("annotation.location"));
+            locValue1.setFieldValue("name", new StringFieldValue("Prinsens Gate"));
+            locAn1 = new Annotation(location, locValue1);
+        }
+        {
+            Struct indValue1 = new Struct(manager.getDataType("annotation.industry"));
+            indValue1.setFieldValue("vertical", new StringFieldValue("Software Services"));
+            indAn1 = new Annotation(industry, indValue1);
+        }
+        {
+            Struct companyValue2 = (Struct) company.getDataType().createFieldValue();
+            companyValue2.setFieldValue("name", new StringFieldValue("Yahoo"));
+            companyValue2.setFieldValue("ceo", new StringFieldValue("Carol Bartz"));
+            companyValue2.setFieldValue("lat", new FloatFieldValue(32.1f));
+            companyValue2.setFieldValue("lon", new FloatFieldValue(-48.44f));
+            companyValue2.setFieldValue("alt", 33.56);
+            companyValue2.setFieldValue("vertical", new StringFieldValue("Research"));
+            compAn2 = new Annotation(company, companyValue2);
+        }
+        {
+            Struct personValue2 = new Struct(manager.getDataType("annotation.person"));
+            personValue2.setFieldValue("name", new StringFieldValue("Kim Johansen"));
+            personAn2 = new Annotation(person, personValue2);
+        }
+        {
+            Struct locValue2 = new Struct(manager.getDataType("annotation.location"));
+            locValue2.setFieldValue("name", new StringFieldValue("RT Nagar"));
+            locAn2 = new Annotation(location, locValue2);
+        }
+        {
+            Struct indValue2 = new Struct(manager.getDataType("annotation.industry"));
+            indValue2.setFieldValue("vertical", new StringFieldValue("Software Consulting"));
+            indAn2 = new Annotation(industry, indValue2);
+        }
+
+        SpanTree tree = new SpanTree("test");
+        SpanList root = (SpanList) tree.getRoot();
+        AlternateSpanList branch = new AlternateSpanList();
+
+        SpanNode span1 = new Span(0, 3);
+        SpanNode span2 = new Span(1, 9);
+        SpanNode span3 = new Span(12, 10);
+
+        SpanNode span11 = new Span(0, 3);
+        SpanNode span22 = new Span(1, 9);
+        SpanNode span33 = new Span(12, 10);
+
+        SpanList alternate1 = new SpanList();
+        alternate1.add(span3);
+        alternate1.add(span2);
+        alternate1.add(span1);
+
+        SpanList alternate2 = new SpanList();
+        alternate2.add(span11);
+        alternate2.add(span22);
+        alternate2.add(span33);
+
+        tree.annotate(span1, compAn1);
+        tree.annotate(span2, personAn1);
+        tree.annotate(span3, locAn1);
+        tree.annotate(span1, indAn1);
+
+        tree.annotate(span11, compAn2);
+        tree.annotate(span22, personAn2);
+        tree.annotate(span33, locAn2);
+        tree.annotate(span11, indAn2);
+
+        List<SpanNode> subtreeList1 = new ArrayList<>();
+        subtreeList1.add(alternate1);
+
+        List<SpanNode> subtreeList2 = new ArrayList<>();
+        subtreeList2.add(alternate2);
+        branch.addChildren(1, subtreeList1, 20.0d);
+        branch.addChildren(2, subtreeList2, 50.0d);
+        root.add(branch);
+
+        StringFieldValue body = (StringFieldValue) document.getFieldValue(document.getDataType().getField("body"));
+        body.setSpanTree(tree);
+        document.setFieldValue(document.getDataType().getField("body"), body);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java
new file mode 100644
index 00000000000..ee448bb79b0
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java
@@ -0,0 +1,123 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;

+

+import com.yahoo.document.Document;

+import com.yahoo.document.DocumentTypeManager;

+import com.yahoo.document.DocumentTypeManagerConfigurer;

+import com.yahoo.document.StructDataType;

+import com.yahoo.document.datatypes.*;

+import com.yahoo.document.serialization.*;

+import com.yahoo.io.GrowableByteBuffer;

+import org.junit.Test;

+

+import java.util.ArrayList;

+import java.util.Collections;

+

+import static org.junit.Assert.*;

+

+public class Bug6394548TestCase {

+    @Test

+    public void testSerializeAndDeserializeMultipleAdjacentStructAnnotations() {

+        DocumentTypeManager manager = new DocumentTypeManager();

+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/java/com/yahoo/document/annotation/documentmanager.6394548.cfg");

+

+        AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();

+        AnnotationType featureSetType = registry.getType("morty.RICK_FEATURESET");

+        assertNotNull(featureSetType);

+

+        Document doc = new Document(manager.getDocumentType("article"), "doc:article:test");

+        StringFieldValue sfv = new StringFieldValue("badger waltz");

+

+        SpanList root = new SpanList();

+        SpanNode node = new Span(0, sfv.getString().length());

+        root.add(node);

+

+        SpanTree tree = new SpanTree("rick_features", root);

+        for (int i = 0; i < 2; ++i) {

+            tree.annotate(createBigFeatureSetAnnotation(featureSetType));

+        }

+

+        sfv.setSpanTree(tree);

+        doc.setFieldValue("title", sfv);

+        System.out.println(doc.toXml());

+        String annotationsBefore = dumpAllAnnotations(tree);

+

+        GrowableByteBuffer buffer = new GrowableByteBuffer();

+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);

+        serializer.write(doc);

+

+        buffer.flip();

+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer);

+        Document doc2 = new Document(deserializer);

+

+        System.out.println(doc2.toXml());

+

+        StringFieldValue readString = (StringFieldValue)doc2.getFieldValue("title");

+        SpanTree readTree = readString.getSpanTree("rick_features");

+        assertNotNull(readTree);

+        String annotationsAfter = dumpAllAnnotations(readTree);

+

+        System.out.println("before:\n" + annotationsBefore);

+        System.out.println("after:\n" + annotationsAfter);

+

+        assertEquals(annotationsBefore, annotationsAfter);

+    }

+

+    private String dumpAllAnnotations(SpanTree tree) {

+        ArrayList<String> tmp = new ArrayList<>();

+        for (Annotation anno : tree) {

+            Struct s = (Struct)anno.getFieldValue();

+            tmp.add(s.toXml());

+        }

+        Collections.sort(tmp);

+        StringBuilder annotations = new StringBuilder();

+        for (String s : tmp) {

+            annotations.append(s);

+        }

+        return annotations.toString();

+    }

+

+    private Annotation createBigFeatureSetAnnotation(AnnotationType featuresetAnno) {

+        StructDataType featuresetType = (StructDataType)featuresetAnno.getDataType();

+        Struct featureset = featuresetType.createFieldValue();

+        System.out.println("featureset type: " + featureset.getDataType().toString());

+

+        MapFieldValue<StringFieldValue, IntegerFieldValue> discreteValued

+                = (MapFieldValue<StringFieldValue, IntegerFieldValue>)featuresetType.getField("discretevaluedfeatures").getDataType().createFieldValue();

+        discreteValued.put(new StringFieldValue("foo"), new IntegerFieldValue(1234));

+        discreteValued.put(new StringFieldValue("bar"), new IntegerFieldValue(567890123));

+        featureset.setFieldValue("discretevaluedfeatures", discreteValued);

+

+        MapFieldValue<StringFieldValue, DoubleFieldValue> realValued

+                = (MapFieldValue<StringFieldValue, DoubleFieldValue>)featuresetType.getField("realvaluedfeatures").getDataType().createFieldValue();

+        realValued.put(new StringFieldValue("foo"), new DoubleFieldValue(0.75));

+        realValued.put(new StringFieldValue("bar"), new DoubleFieldValue(1.5));

+        featureset.setFieldValue("realvaluedfeatures", realValued);

+

+        Array<StringFieldValue> nested = (Array<StringFieldValue>)featureset.getField("foo10").getDataType().createFieldValue();

+        nested.add(new StringFieldValue("baz"));

+        nested.add(new StringFieldValue("blargh"));

+        featureset.setFieldValue("foo10", nested);

+

+        featureset.setFieldValue("foo1", new StringFieldValue("asdf"));

+        featureset.setFieldValue("foo4", new StringFieldValue("qwerty"));

+        featureset.setFieldValue("foo2", new IntegerFieldValue(555));

+        featureset.setFieldValue("foo2", new IntegerFieldValue(8));

+        featureset.setFieldValue("foo7", new IntegerFieldValue(1337));

+        featureset.setFieldValue("foo8", new IntegerFieldValue(967867));

+        featureset.setFieldValue("foo9", new DoubleFieldValue(123.45));

+

+        Array<StringFieldValue> attributes = (Array<StringFieldValue>)featureset.getField("foo6").getDataType().createFieldValue();

+        attributes.add(new StringFieldValue("adam"));

+        attributes.add(new StringFieldValue("jamie"));

+        attributes.add(new StringFieldValue("grant"));

+        attributes.add(new StringFieldValue("tory"));

+        attributes.add(new StringFieldValue("kari"));

+        featureset.setFieldValue("variantattribute", attributes);

+

+        Annotation anno = new Annotation(featuresetAnno);

+        anno.setFieldValue(featureset);

+

+        return anno;

+    }

+}

diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java
new file mode 100644
index 00000000000..d34d0da8e82
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;

+

+import com.yahoo.document.DataType;

+import com.yahoo.document.DocumentTypeManager;

+import com.yahoo.document.Field;

+import com.yahoo.document.StructDataType;

+import com.yahoo.document.datatypes.StringFieldValue;

+import com.yahoo.document.datatypes.Struct;

+import com.yahoo.document.serialization.*;

+import com.yahoo.io.GrowableByteBuffer;

+import org.junit.Before;

+import org.junit.Test;

+import static org.junit.Assert.*;

+

+public class Bug6425939TestCase {

+    private DocumentTypeManager man;

+    private StructDataType person;

+    private AnnotationType personA;

+

+    @Before

+    public void setUp() {

+        man = new DocumentTypeManager();

+

+        person = new StructDataType("personStruct");

+        person.addField(new Field("foo", DataType.STRING));

+        person.addField(new Field("bar", DataType.INT));

+        man.register(person);

+

+        personA = new AnnotationType("person", person);

+        man.getAnnotationTypeRegistry().register(personA);

+    }

+

+    @Test

+    public void canDeserializeAnnotationsOnZeroLengthStrings() {

+        StringFieldValue emptyString = new StringFieldValue("");

+        emptyString.setSpanTree(createSpanTree());

+

+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);

+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);

+        Field strField = new Field("flarn", DataType.STRING);

+        serializer.write(strField, emptyString);

+        buffer.flip();

+

+        // Should not throw exception if bug 6425939 is fixed:

+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);

+        StringFieldValue deserializedString = new StringFieldValue();

+        deserializer.read(strField, deserializedString);

+

+        assertEquals("", deserializedString.getString());

+        SpanTree readTree = deserializedString.getSpanTree("SpanTree1");

+        assertNotNull(readTree);

+    }

+

+    private SpanTree createSpanTree() {

+        SpanList root = new SpanList();

+        SpanTree tree = new SpanTree("SpanTree1", root);

+        SpanNode node = new Span(0, 0);

+        Struct ps = new Struct(person);

+        ps.setFieldValue("foo", "epic badger");

+        ps.setFieldValue("bar", 54321);

+        tree.annotate(node, new Annotation(personA, ps));

+        root.add(node);

+        return tree;

+    }

+}

diff --git a/document/src/test/java/com/yahoo/document/annotation/DocTestCase.java b/document/src/test/java/com/yahoo/document/annotation/DocTestCase.java
new file mode 100644
index 00000000000..565cbd6c370
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/DocTestCase.java
@@ -0,0 +1,425 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * Contains code snippets that are used in the documentation. Not really a test case.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class DocTestCase extends junit.framework.TestCase {
+
+    private class Processing {
+        private Service getService() {
+            return null;
+        }
+    }
+
+    private class Service {
+        private DocumentTypeManager getDocumentTypeManager() {
+            return null;
+        }
+    }
+
+    private Processing processing = null;
+
+
+    public void testSimple1() {
+        StringFieldValue text = new StringFieldValue("<html><head><title>Diary</title></head><body>I live in San Francisco</body></html>");
+                                                    //012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+        SpanList root = new SpanList();
+        root.add(new Span(0, 19))
+                .add(new Span(19, 5))
+                .add(new Span(24, 21))
+                .add(new Span(45, 23))
+                .add(new Span(68, 14));
+
+        SpanTree tree = new SpanTree("html", root);
+        text.setSpanTree(tree);
+    }
+
+    public void simple2() {
+        //the following line works inside process(Document, Arguments, Processing) in a DocumentProcessor
+        AnnotationTypeRegistry atr = processing.getService().getDocumentTypeManager().getAnnotationTypeRegistry();
+
+        StringFieldValue text = new StringFieldValue("<html><head><title>Diary</title></head><body>I live in San Francisco</body></html>");
+                                                    //012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+        AnnotationType textType = atr.getType("text");
+        AnnotationType markup = atr.getType("markup");
+
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("html", root);
+
+
+        Span span1 = new Span(0, 19);
+        root.add(span1);
+        tree.annotate(span1, markup);
+
+        Span span2 = new Span(19, 5);
+        root.add(span2);
+        tree.annotate(span2, textType);
+
+        Span span3 = new Span(24, 21);
+        root.add(span3);
+        tree.annotate(span3, markup);
+
+        Span span4 = new Span(45, 23);
+        root.add(span4);
+        tree.annotate(span4, textType);
+
+        Span span5 = new Span(68, 14);
+        root.add(span5);
+        tree.annotate(span5, markup);
+
+
+        text.setSpanTree(tree);
+    }
+
+    public void simple3() {
+        //the following line works inside process(Document, Arguments, Processing) in a DocumentProcessor
+        AnnotationTypeRegistry atr = processing.getService().getDocumentTypeManager().getAnnotationTypeRegistry();
+
+        StringFieldValue text = new StringFieldValue("<html><head><title>Diary</title></head><body>I live in San Francisco</body></html>");
+                                                    //012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("html", root);
+
+        AnnotationType textType = atr.getType("text");
+        AnnotationType beginTag = atr.getType("begintag");
+        AnnotationType endTag = atr.getType("endtag");
+        AnnotationType bodyType = atr.getType("body");
+        AnnotationType headerType = atr.getType("header");
+
+        SpanList header = new SpanList();
+        {
+            Span span1 = new Span(6, 6);
+            Span span2 = new Span(12, 7);
+            Span span3 = new Span(19, 5);
+            Span span4 = new Span(24, 8);
+            Span span5 = new Span(32, 7);
+            header.add(span1)
+                    .add(span2)
+                    .add(span3)
+                    .add(span4)
+                    .add(span5);
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, beginTag)
+                    .annotate(span3, textType)
+                    .annotate(span4, endTag)
+                    .annotate(span5, endTag)
+                    .annotate(header, headerType);
+        }
+
+        SpanList body = new SpanList();
+        {
+            Span span1 = new Span(39, 6);
+            Span span2 = new Span(45, 23);
+            Span span3 = new Span(68, 7);
+            body.add(span1)
+                    .add(span2)
+                    .add(span3);
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, textType)
+                    .annotate(span3, endTag)
+                    .annotate(body, bodyType);
+        }
+
+        {
+            Span span1 = new Span(0, 6);
+            Span span2 = new Span(75, 7);
+            root.add(span1)
+                    .add(header)
+                    .add(body)
+                    .add(span2);
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, endTag);
+        }
+
+        text.setSpanTree(tree);
+    }
+
+
+    public void simple4() {
+        //the following line works inside process(Document, Arguments, Processing) in a DocumentProcessor
+        AnnotationTypeRegistry atr = processing.getService().getDocumentTypeManager().getAnnotationTypeRegistry();
+
+        StringFieldValue text = new StringFieldValue("<html><head><title>Diary</title></head><body>I live in San Francisco</body></html>");
+                                                    //012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("html", root);
+
+        AnnotationType textType = atr.getType("text");
+        AnnotationType beginTag = atr.getType("begintag");
+        AnnotationType endTag = atr.getType("endtag");
+        AnnotationType bodyType = atr.getType("body");
+        AnnotationType headerType = atr.getType("header");
+        AnnotationType cityType = atr.getType("city");
+
+        Struct position = (Struct) cityType.getDataType().createFieldValue();
+        position.setFieldValue("latitude", 37.774929);
+        position.setFieldValue("longitude", -122.419415);
+        Annotation city = new Annotation(cityType, position);
+
+        SpanList header = new SpanList();
+        {
+            Span span1 = new Span(6, 6);
+            Span span2 = new Span(12, 7);
+            Span span3 = new Span(19, 5);
+            Span span4 = new Span(24, 8);
+            Span span5 = new Span(32, 7);
+            header.add(span1)
+                    .add(span2)
+                    .add(span3)
+                    .add(span4)
+                    .add(span5);
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, beginTag)
+                    .annotate(span3, textType)
+                    .annotate(span4, endTag)
+                    .annotate(span4, endTag)
+                    .annotate(header, headerType);
+        }
+
+        SpanList textNode = new SpanList();
+        {
+            Span span1 = new Span(45, 10);
+            Span span2 = new Span(55, 13);
+            textNode.add(span1)
+                    .add(span2);
+            tree.annotate(span2, city)
+                    .annotate(textNode, textType);
+        }
+
+        SpanList body = new SpanList();
+        {
+            Span span1 = new Span(39, 6);
+            Span span2 = new Span(68, 7);
+            body.add(span1)
+                    .add(textNode)
+                    .add(span2);
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, endTag)
+                    .annotate(body, bodyType);
+        }
+
+        {
+            Span span1 = new Span(0, 6);
+            Span span2 = new Span(75, 7);
+            root.add(span1)
+                    .add(header)
+                    .add(body)
+                    .add(span2);
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, endTag);
+        }
+
+        text.setSpanTree(tree);
+    }
+
+
+    public void simple5() {
+        //the following two lines work inside process(Document, Arguments, Processing) in a DocumentProcessor
+        DocumentTypeManager dtm = processing.getService().getDocumentTypeManager();
+        AnnotationTypeRegistry atr = dtm.getAnnotationTypeRegistry();
+
+        StringFieldValue text = new StringFieldValue("<body><p>I live in San </p>Francisco</body>");
+                                                    //012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("html", root);
+
+        StructDataType positionType = (StructDataType) dtm.getDataType("position");
+
+        AnnotationType textType = atr.getType("text");
+        AnnotationType beginTag = atr.getType("begintag");
+        AnnotationType endTag = atr.getType("endtag");
+        AnnotationType bodyType = atr.getType("body");
+        AnnotationType paragraphType = atr.getType("paragraph");
+        AnnotationType cityType = atr.getType("city");
+
+        Struct position = new Struct(positionType);
+        position.setFieldValue("latitude", 37.774929);
+        position.setFieldValue("longitude", -122.419415);
+
+        Annotation sanAnnotation = new Annotation(textType);
+        Annotation franciscoAnnotation = new Annotation(textType);
+
+        Struct positionWithRef = (Struct) cityType.getDataType().createFieldValue();
+        positionWithRef.setFieldValue("position", position);
+
+        Field referencesField = ((StructDataType) cityType.getDataType()).getField("references");
+        Array<FieldValue> refList = new Array<FieldValue>(referencesField.getDataType());
+        AnnotationReferenceDataType annRefType = (AnnotationReferenceDataType) ((ArrayDataType) referencesField.getDataType()).getNestedType();
+        refList.add(new AnnotationReference(annRefType, sanAnnotation));
+        refList.add(new AnnotationReference(annRefType, franciscoAnnotation));
+        positionWithRef.setFieldValue(referencesField, refList);
+
+        Annotation city = new Annotation(cityType, positionWithRef);
+
+        SpanList paragraph = new SpanList();
+        {
+            Span span1 = new Span(6, 3);
+            Span span2 = new Span(9, 10);
+            Span span3 = new Span(19, 4);
+            Span span4 = new Span(23, 4);
+            paragraph.add(span1)
+                    .add(span2)
+                    .add(span3)
+                    .add(span4);
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, textType)
+                    .annotate(span3, sanAnnotation)
+                    .annotate(span4, endTag)
+                    .annotate(paragraph, paragraphType);
+        }
+
+        {
+            Span span1 = new Span(0, 6);
+            Span span2 = new Span(27, 9);
+            Span span3 = new Span(36, 8);
+            root.add(span1)
+                    .add(paragraph)
+                    .add(span2)
+                    .add(span3);
+
+            tree.annotate(span1, beginTag)
+                    .annotate(span2, franciscoAnnotation)
+                    .annotate(span3, endTag)
+                    .annotate(root, bodyType)
+                    .annotate(city);
+        }
+
+        text.setSpanTree(tree);
+    }
+
+    public void simple6() {
+        StringFieldValue text = new StringFieldValue("<html><head><title>Diary</title></head><body>I live in San Francisco</body></html>");
+                                                    //012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+
+        SpanTree tree = text.getSpanTree("html");
+        SpanList root = (SpanList) tree.getRoot();
+        //TODO: Note that the above could have been a Span or an AlternateSpanList!
+
+        ListIterator<SpanNode> nodeIt = root.childIterator();
+
+        AnnotationType beginTag = new AnnotationType("begintag");
+        AnnotationType endTag = new AnnotationType("endtag");
+
+
+        while (nodeIt.hasNext()) {
+            SpanNode node = nodeIt.next();
+            boolean nodeHadMarkupAnnotation = removeMarkupAnnotation(tree, node);
+            if (nodeHadMarkupAnnotation) {
+                nodeIt.remove();
+                List<Span> replacementNodes = analyzeMarkup(tree, node, text, beginTag, endTag);
+                for (SpanNode repl : replacementNodes) {
+                    nodeIt.add(repl);
+                }
+            }
+        }
+    }
+
+    /**
+     * Removes annotations of type 'markup' from the given node.
+     *
+     * @param tree the tree to remove annotations from
+     * @param node the node to remove annotations of type 'markup' from
+     * @return true if the given node had 'markup' annotations, false otherwise
+     */
+    private boolean removeMarkupAnnotation(SpanTree tree, SpanNode node) {
+        //get iterator over all annotations on this node:
+        Iterator<Annotation> annotationIt = tree.iterator(node);
+
+        while (annotationIt.hasNext()) {
+            Annotation annotation = annotationIt.next();
+            if (annotation.getType().getName().equals("markup")) {
+                //this node has an annotation of type markup, remove it:
+                annotationIt.remove();
+                //return true, this node had a markup annotation:
+                return true;
+            }
+        }
+        //this node did not have a markup annotation:
+        return false;
+    }
+
+    /**
+     * NOTE: This method is provided only for completeness. It analyzes spans annotated with
+     * &quot;markup&quot;, and splits them into several shorter spans annotated with &quot;begintag&quot;
+     * and &quot;endtag&quot;.
+     *
+     * @param tree       the span tree to annotate into
+     * @param input      a SpanNode that is annotated with &quot;markup&quot;.
+     * @param text       the text that the SpanNode covers
+     * @param beginTag   the type to use for begintag annotations
+     * @param endTagType the type to use for endtag annotations
+     * @return a list of new spans to replace the input
+     */
+    private List<Span> analyzeMarkup(SpanTree tree, SpanNode input, StringFieldValue text,
+                                     AnnotationType beginTag, AnnotationType endTagType) {
+        //we know that this node is annotated with "markup"
+        String coveredText = input.getText(text.getString()).toString();
+        int spanOffset = input.getFrom();
+        int tagStart = -1;
+        boolean endTag = false;
+        List<Span> tags = new ArrayList<Span>();
+        for (int i = 0; i < coveredText.length(); i++) {
+            if (coveredText.charAt(i) == '<') {
+                //we're in a tag
+                tagStart = i;
+                continue;
+            }
+            if (coveredText.charAt(i) == '>' && tagStart > -1) {
+                Span span = new Span(spanOffset + tagStart, (i + 1) - tagStart);
+                tags.add(span);
+                if (endTag) {
+                    tree.annotate(span, endTagType);
+                } else {
+                    tree.annotate(span, beginTag);
+                }
+                tagStart = -1;
+            }
+            if (tagStart > -1 && i == (tagStart + 1)) {
+                if (coveredText.charAt(i) == '/') {
+                    endTag = true;
+                } else {
+                    endTag = false;
+                }
+            }
+        }
+        return tags;
+    }
+
+    public void simple7() {
+        StringFieldValue text = new StringFieldValue("<html><head><title>Diary</title></head><body>I live in San Francisco</body></html>");
+                                                    //012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+
+        SpanTree tree = text.getSpanTree("html");
+
+        Iterator<Annotation> annotationIt = tree.iterator();
+
+        while (annotationIt.hasNext()) {
+            Annotation annotation = annotationIt.next();
+            if (annotation.getType().getName().equals("markup")) {
+                //we have an annotation of type markup, remove it:
+                annotationIt.remove();
+            }
+        }
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/DummySpanNodeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/DummySpanNodeTestCase.java
new file mode 100644
index 00000000000..f238b906f6b
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/DummySpanNodeTestCase.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.10
+ */
+public class DummySpanNodeTestCase {
+
+    @Test
+    public void basic() {
+        DummySpanNode node = DummySpanNode.INSTANCE;
+        assertThat(node.getFrom(), is(0));
+        assertThat(node.getTo(), is(0));
+        assertThat(node.getLength(), is(0));
+        assertThat(node.getText("baba"), nullValue());
+        assertThat(node.isLeafNode(), is(true));
+        assertThat(node.childIterator().hasNext(), is(false));
+        assertThat(node.childIterator().hasPrevious(), is(false));
+        assertThat(node.childIteratorRecursive().hasNext(), is(false));
+        assertThat(node.childIteratorRecursive().hasPrevious(), is(false));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/IndexKeyAnnotationTypeSpanTreeAdvTest.java b/document/src/test/java/com/yahoo/document/annotation/IndexKeyAnnotationTypeSpanTreeAdvTest.java
new file mode 100644
index 00000000000..790813abe73
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/IndexKeyAnnotationTypeSpanTreeAdvTest.java
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexKeyAnnotationTypeSpanTreeAdvTest extends SpanTreeAdvTest {
+    @Override
+    public void populateSpanTree() {
+        super.populateSpanTree();
+        tree.createIndex(SpanTree.IndexKey.ANNOTATION_TYPE);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/IndexKeyAnnotationTypeSpanTreeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/IndexKeyAnnotationTypeSpanTreeTestCase.java
new file mode 100644
index 00000000000..8bad1bdc37d
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/IndexKeyAnnotationTypeSpanTreeTestCase.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexKeyAnnotationTypeSpanTreeTestCase extends SpanTreeTestCase {
+
+    @Override
+    public void setUp() {
+        super.setUp();
+        tree.createIndex(SpanTree.IndexKey.ANNOTATION_TYPE);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanNodeSpanTreeAdvTest.java b/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanNodeSpanTreeAdvTest.java
new file mode 100644
index 00000000000..49c34d1fee4
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanNodeSpanTreeAdvTest.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexKeySpanNodeSpanTreeAdvTest extends SpanTreeAdvTest {
+
+    @Override
+    public void populateSpanTree() {
+        super.populateSpanTree();
+        tree.createIndex(SpanTree.IndexKey.SPAN_NODE);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanNodeSpanTreeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanNodeSpanTreeTestCase.java
new file mode 100644
index 00000000000..dc685f3079d
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanNodeSpanTreeTestCase.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexKeySpanNodeSpanTreeTestCase extends SpanTreeTestCase {
+
+    @Override
+    public void setUp() {
+        super.setUp();
+        tree.createIndex(SpanTree.IndexKey.SPAN_NODE);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanTreeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanTreeTestCase.java
new file mode 100644
index 00000000000..243367e4f54
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/IndexKeySpanTreeTestCase.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexKeySpanTreeTestCase {
+
+    @Test
+    public void testIndexKeys() throws Exception {
+        SpanTree tree = new SpanTree("something");
+        assertThat(tree.getCurrentIndexes().isEmpty(), is(true));
+
+        tree.createIndex(SpanTree.IndexKey.SPAN_NODE);
+        assertThat(tree.getCurrentIndexes().size(), is(1));
+        assertThat(tree.getCurrentIndexes().iterator().next(), is(SpanTree.IndexKey.SPAN_NODE));
+
+        tree.clearIndexes();
+        assertThat(tree.getCurrentIndexes().isEmpty(), is(true));
+
+        tree.createIndex(SpanTree.IndexKey.ANNOTATION_TYPE);
+        assertThat(tree.getCurrentIndexes().size(), is(1));
+        assertThat(tree.getCurrentIndexes().iterator().next(), is(SpanTree.IndexKey.ANNOTATION_TYPE));
+
+        tree.clearIndexes();
+        assertThat(tree.getCurrentIndexes().isEmpty(), is(true));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSwitchIndexes()  {
+        SpanTree tree = new SpanTree("something");
+        assertThat(tree.getCurrentIndexes().isEmpty(), is(true));
+        tree.createIndex(SpanTree.IndexKey.SPAN_NODE);
+        tree.createIndex(SpanTree.IndexKey.ANNOTATION_TYPE);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSwitchIndexes2()  {
+        SpanTree tree = new SpanTree("something");
+        assertThat(tree.getCurrentIndexes().isEmpty(), is(true));
+        tree.createIndex(SpanTree.IndexKey.ANNOTATION_TYPE);
+        tree.createIndex(SpanTree.IndexKey.SPAN_NODE);
+    }
+
+    @Test
+    public void testSwitchIndexes3()  {
+        SpanTree tree = new SpanTree("something");
+        assertThat(tree.getCurrentIndexes().isEmpty(), is(true));
+        tree.createIndex(SpanTree.IndexKey.ANNOTATION_TYPE);
+        tree.clearIndex(SpanTree.IndexKey.SPAN_NODE);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/PeekableListIteratorTestCase.java b/document/src/test/java/com/yahoo/document/annotation/PeekableListIteratorTestCase.java
new file mode 100644
index 00000000000..d63cd324033
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/PeekableListIteratorTestCase.java
@@ -0,0 +1,335 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+/**
+ * Tests that PeekableListIterator behaves as expected.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class PeekableListIteratorTestCase extends junit.framework.TestCase {
+    private List<String> strings;
+
+    @Override
+    public void setUp() {
+        strings = new ArrayList<String>();
+        strings.add("0");
+        strings.add("1");
+        strings.add("2");
+        strings.add("3");
+        strings.add("4");
+        strings.add("5");
+    }
+
+    public void testSimpleListIterator() {
+        ListIterator<String> it = strings.listIterator();
+        assertEquals("0", it.next());
+        assertEquals("1", it.next());
+        //cursor is before "2"
+        assertEquals("1", it.previous());
+        //cursor is before "1"
+        assertEquals("0", it.previous());
+        //cursor is before "0"
+        //removing 0, it's the last one returned by next() or previous():
+        it.remove();
+        assertEquals("1", it.next());
+    }
+
+    public void testVarious() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertTrue(it.hasNext());
+        assertEquals("0", it.peek());
+        assertTrue(it.hasNext());
+        assertEquals("0", it.next());
+        assertEquals("1", it.next());
+        assertEquals("2", it.next());
+        assertEquals("3", it.peek());
+        assertEquals("3", it.peek());
+        assertEquals("3", it.peek());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertEquals("3", it.next());
+        //cursor is now before 4
+        it.add("banana");
+        it.add("apple");
+        //added before 4
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertTrue(it.hasNext());
+        assertEquals("4", it.peek());
+        assertEquals("4", it.peek());
+        assertEquals("4", it.peek());
+        assertEquals("4", it.peek());
+        //removing 3
+        it.remove();
+        assertEquals("4", it.peek());
+        assertEquals("4", it.next());
+        //replacing 4 with orange:
+        it.set("orange");
+        assertEquals("5", it.peek());
+        assertEquals("5", it.next());
+        assertFalse(it.hasNext());
+        assertNull(it.peek());
+        try {
+            it.next();
+            fail("Shouldn't have worked.");
+        } catch (NoSuchElementException nsee) {
+            // empty
+        }
+    }
+
+    public void testRemoveFirst() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+        try {
+            it.remove();  //this shouldn't work
+            fail("shouldn't work");
+        } catch (IllegalStateException ise) {
+            //nada
+        }
+
+        assertEquals("0", it.next());
+        it.remove();
+
+        assertEquals(5, strings.size());
+        assertEquals("1", strings.get(0));
+    }
+
+    public void testPeekThenRemove() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        it.peek();
+        try {
+            it.remove();  //this should not work!
+            fail("should have gotten exception");
+        } catch (IllegalStateException ise) {
+            //niks
+        }
+
+        assertEquals("0", it.next());
+        it.remove();  //this should work
+        assertEquals(5, strings.size());
+        assertEquals("1", strings.get(0));
+        assertEquals("2", strings.get(1));
+        assertEquals("3", strings.get(2));
+        assertEquals("4", strings.get(3));
+        assertEquals("5", strings.get(4));
+    }
+
+    public void testPeekNextRemove() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.peek());
+        assertEquals("0", it.next());
+        it.remove();
+        assertEquals(5, strings.size());
+        assertEquals("1", strings.get(0));
+        assertEquals("2", strings.get(1));
+        assertEquals("3", strings.get(2));
+        assertEquals("4", strings.get(3));
+        assertEquals("5", strings.get(4));
+    }
+
+    public void testPeekNextPeekRemove() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.peek());
+        assertEquals("0", it.next());
+        assertEquals("1", it.peek());
+        it.remove();
+        assertEquals(5, strings.size());
+        assertEquals("1", strings.get(0));
+        assertEquals("2", strings.get(1));
+        assertEquals("3", strings.get(2));
+        assertEquals("4", strings.get(3));
+        assertEquals("5", strings.get(4));
+    }
+
+    public void testPeekNextNextRemove() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.peek());
+        assertEquals("0", it.next());
+        assertEquals("1", it.next());
+        it.remove();
+        assertEquals(5, strings.size());
+        assertEquals("0", strings.get(0));
+        assertEquals("2", strings.get(1));
+        assertEquals("3", strings.get(2));
+        assertEquals("4", strings.get(3));
+        assertEquals("5", strings.get(4));
+    }
+
+    public void testNextPeekRemove() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.next());
+        assertEquals("1", it.peek());
+        it.remove();
+        assertEquals(5, strings.size());
+        assertEquals("1", strings.get(0));
+        assertEquals("2", strings.get(1));
+        assertEquals("3", strings.get(2));
+        assertEquals("4", strings.get(3));
+        assertEquals("5", strings.get(4));
+    }
+
+    public void testAddSimple() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        it.add("a");
+        it.add("b");
+        it.add("c");
+        assertEquals("0", it.next());
+
+        assertEquals(9, strings.size());
+        assertEquals("a", strings.get(0));
+        assertEquals("b", strings.get(1));
+        assertEquals("c", strings.get(2));
+        assertEquals("0", strings.get(3));
+        assertEquals("1", strings.get(4));
+        assertEquals("2", strings.get(5));
+        assertEquals("3", strings.get(6));
+        assertEquals("4", strings.get(7));
+        assertEquals("5", strings.get(8));
+    }
+
+    public void testPeekAdd() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        it.peek();
+        it.add("a");
+        it.add("b");
+        it.add("c");
+        assertEquals("0", it.next());
+
+        assertEquals(9, strings.size());
+        assertEquals("a", strings.get(0));
+        assertEquals("b", strings.get(1));
+        assertEquals("c", strings.get(2));
+        assertEquals("0", strings.get(3));
+        assertEquals("1", strings.get(4));
+        assertEquals("2", strings.get(5));
+        assertEquals("3", strings.get(6));
+        assertEquals("4", strings.get(7));
+        assertEquals("5", strings.get(8));
+    }
+
+
+    public void testSetFirst() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+        try {
+            it.set("kjell");  //this shouldn't work
+            fail("shouldn't work");
+        } catch (IllegalStateException ise) {
+            //nada
+        }
+
+        assertEquals("0", it.next());
+        it.set("elvis");
+
+        assertEquals(6, strings.size());
+        assertEquals("elvis", strings.get(0));
+    }
+
+    public void testPeekThenSet() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        it.peek();
+        try {
+            it.set("elvis");  //this should not work!
+            fail("should have gotten exception");
+        } catch (IllegalStateException ise) {
+            //niks
+        }
+
+        assertEquals("0", it.next());
+        it.set("presley");  //this should work
+        assertEquals(6, strings.size());
+        assertEquals("presley", strings.get(0));
+        assertEquals("1", strings.get(1));
+        assertEquals("2", strings.get(2));
+        assertEquals("3", strings.get(3));
+        assertEquals("4", strings.get(4));
+        assertEquals("5", strings.get(5));
+    }
+
+    public void testPeekNextSet() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.peek());
+        assertEquals("0", it.next());
+        it.set("buddy");
+        assertEquals(6, strings.size());
+        assertEquals("buddy", strings.get(0));
+        assertEquals("1", strings.get(1));
+        assertEquals("2", strings.get(2));
+        assertEquals("3", strings.get(3));
+        assertEquals("4", strings.get(4));
+        assertEquals("5", strings.get(5));
+    }
+
+    public void testPeekNextPeekSet() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.peek());
+        assertEquals("0", it.next());
+        assertEquals("1", it.peek());
+        it.set("holly");
+        assertEquals(6, strings.size());
+        assertEquals("holly", strings.get(0));
+        assertEquals("1", strings.get(1));
+        assertEquals("2", strings.get(2));
+        assertEquals("3", strings.get(3));
+        assertEquals("4", strings.get(4));
+        assertEquals("5", strings.get(5));
+    }
+
+    public void testPeekNextNextSet() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.peek());
+        assertEquals("0", it.next());
+        assertEquals("1", it.next());
+        it.set("jerry");
+        assertEquals(6, strings.size());
+        assertEquals("0", strings.get(0));
+        assertEquals("jerry", strings.get(1));
+        assertEquals("2", strings.get(2));
+        assertEquals("3", strings.get(3));
+        assertEquals("4", strings.get(4));
+        assertEquals("5", strings.get(5));
+    }
+
+    public void testNextPeekSet() {
+        PeekableListIterator<String> it = new PeekableListIterator<String>(strings.listIterator());
+
+        assertEquals("0", it.next());
+        assertEquals("1", it.peek());
+        it.set("lee");
+        assertEquals(6, strings.size());
+        assertEquals("lee", strings.get(0));
+        assertEquals("1", strings.get(1));
+        assertEquals("2", strings.get(2));
+        assertEquals("3", strings.get(3));
+        assertEquals("4", strings.get(4));
+        assertEquals("5", strings.get(5));
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanListAdvTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanListAdvTestCase.java
new file mode 100644
index 00000000000..c43385f1572
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SpanListAdvTestCase.java
@@ -0,0 +1,284 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Collection;
+import java.util.ListIterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:mpraveen@yahoo-inc.com">Praveen Mohan</a>
+ *
+ * This test covers all possible scenarios in a SpanList.
+ *
+ */
+
+public class SpanListAdvTestCase {
+
+    private boolean debug = false;
+
+    private AnnotationType at1 = new AnnotationType("person", DataType.STRING);
+    private AnnotationType at2 = new AnnotationType("street", DataType.STRING);
+    private AnnotationType at3 = new AnnotationType("city", DataType.STRING);
+
+    private SpanList root = new SpanList();
+    public SpanTree tree = new SpanTree("test", root);
+
+    private SpanNode span1, span2, span3;
+    private SpanNode span11, span22, span33;
+    private SpanNode span111, span222, span333;
+    private SpanList alternate1, alternate2, alternate3;
+
+    @Before
+    public void buildTree() {
+        tree.cleanup();
+        span1 = new Span(0, 3);
+        span2 = new Span(4, 6);
+        span3 = new Span(6, 10);
+
+        span11 = new Span(0, 2);
+        span22 = new Span(2, 10);
+        span33 = new Span(12, 10);
+
+        span111 = new Span(5, 10);
+        span222 = new Span(15, 5);
+        span333 = new Span(20, 10);
+
+        alternate1 = new SpanList();
+        alternate1.add(span3);
+        alternate1.add(span2);
+        alternate1.add(span1);
+
+        alternate2 = new SpanList();
+        alternate2.add(span11);
+        alternate2.add(span22);
+        alternate2.add(span33);
+
+        alternate3 = new SpanList();
+        alternate3.add(span111);
+        alternate3.add(span222);
+
+        root.add(span333);
+
+        tree.annotate(span1, at1);
+        tree.annotate(span2, at2);
+        tree.annotate(span3, at3);
+
+        tree.annotate(span11, at1);
+        tree.annotate(span22, at2);
+        tree.annotate(span33, at3);
+
+        tree.annotate(span111, at1);
+        tree.annotate(span222, at2);
+        tree.annotate(span333, at3);
+
+        alternate1.add(alternate3);
+
+        root.add(alternate1);
+        root.add(alternate2);
+    }
+
+    @Test
+    public void assertTree() {
+
+        if (debug) {
+            consumeAnnotations((SpanList)tree.getRoot());
+        }
+
+        assertEquals(0, root.getFrom());
+        assertEquals(30, root.getTo());
+        assertEquals(0, alternate1.getFrom());
+        assertEquals(20, alternate1.getTo());
+        assertEquals(0, alternate2.getFrom());
+        assertEquals(22, alternate2.getTo());
+        assertEquals(5, alternate3.getFrom());
+        assertEquals(20, alternate3.getTo());
+        assertFalse(root.numChildren() != 3 || alternate1.numChildren() != 4 || alternate2.numChildren() != 3 || alternate3.numChildren() != 2);
+
+        ArrayList<SpanNode> al = new ArrayList<SpanNode>();
+        al.add(span333);
+        al.add(alternate1);
+        al.add(alternate2);
+        ListIterator<SpanNode> iter = root.childIterator();
+        while (iter.hasNext()) {
+            SpanNode sn = iter.next();
+            int i = 0;
+            for (i = 0; i < al.size(); i ++) {
+                if (sn == al.get(i)) {
+                    break;
+                }
+            }
+            assertFalse(i >= al.size());
+        }
+
+        iter = root.childIteratorRecursive();
+        boolean nodeFound = false;
+        while (iter.hasNext()) {
+            SpanNode sn = iter.next();
+            if (sn == span222 || sn == span111) {
+                nodeFound = true;
+                break;
+            }
+        }
+        assertTrue(nodeFound);
+
+        alternate1.sortChildren();
+        SpanNode ssn = new Span(2, 1);
+        alternate1.add(ssn);
+        alternate1.sortChildren();
+
+        iter = alternate1.childIterator();
+        int from = -1;
+        while (iter.hasNext()) {
+            SpanNode sn = iter.next();
+            if (from == -1) {
+                from = sn.getFrom();
+                continue;
+            }
+            assertFalse(sn.getFrom() < from);
+            from = sn.getFrom();
+        }
+        alternate1.remove(ssn);
+
+        SpanList sl = alternate3.remove(span111);
+        assertFalse (sl != alternate3);
+        assertFalse(span111.isValid());
+
+        alternate1.remove(alternate3);
+        assertFalse(alternate3.isValid());
+        assertFalse(span222.isValid());
+
+        int noofChild = alternate1.numChildren();
+        for (int i = 0; i < alternate1.numChildren(); ) {
+            alternate1.remove(i);
+        }
+        assertFalse(alternate1.numChildren() != 0);
+
+        int noAnnotations = tree.numAnnotations();
+        tree.cleanup();
+        assertFalse(tree.numAnnotations() != (noAnnotations - 5));
+
+        root.clearChildren();
+        tree.cleanup();
+        assertFalse(tree.numAnnotations() != 0);
+
+        sl = new SpanList(alternate1);
+        root.add(sl);
+        assertFalse(root.getFrom() != -1 || root.getTo() != -1);
+
+        SpanNode newSpan1 = new Span(0, 10);
+        SpanNode newSpan2 = new Span(12, 8);
+        alternate3 = new SpanList();
+        alternate3.add(newSpan1);
+        alternate3.add(newSpan2);
+        alternate2.add(alternate3);
+
+        SpanList newA2 = new SpanList(alternate2);
+        root.add(newA2);
+        assertFalse(root.getFrom() != newA2.getFrom() || root.getTo() != newA2.getTo());
+        assertFalse(newA2.numChildren() != 4);
+    }
+
+    @After
+    public void removeTree() {
+        tree = null;
+    }
+
+
+    public void consumeAnnotations(SpanList root) {
+        if (debug) System.out.println("\n\nSpanList: [" + root.getFrom() + ", " + root.getTo() + "] num Children: " + root.numChildren());
+        if (debug) System.out.println("-------------------");
+        Iterator<SpanNode> childIterator = root.childIterator();
+		while (childIterator.hasNext()) {
+			SpanNode node = childIterator.next();
+            //System.out.println("Span Node: " + node); // + " Span Text: " + node.getText(fieldValStr));
+            if (debug) System.out.println("\n\nSpan Node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+            if (node instanceof AlternateSpanList) {
+                parseAlternateLists((AlternateSpanList)node);
+                if (debug) System.out.println("---- Alternate SpanList complete ---");
+            } else if (node instanceof SpanList) {
+                if (debug) System.out.println("Encountered another span list");
+                SpanList spl = (SpanList) node;
+                ListIterator<SpanNode> lli = spl.childIterator();
+                while (lli.hasNext()) System.out.print(" " + lli.next() + " ");
+                consumeAnnotations((SpanList) node);
+            } else {
+       			if (debug) System.out.println("\nGetting annotations for this span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+   	    	    getAnnotationsForNode(node);
+            }
+		}
+        if (debug) System.out.println("\nGetting annotations for the SpanList itself : [" + root.getFrom() + ", " + root.getTo() + "] ");
+        getAnnotationsForNode(root);
+    }
+
+    public void parseAlternateLists(AlternateSpanList aspl) {
+        int no = aspl.getNumSubTrees();
+        if (debug) System.out.println("Parsing Alternate span list. No of subtrees: " + no);
+        int ctr = 0;
+        while (ctr < no) {
+            if (debug) System.out.println("\nSubTree: " + ctr + " probability: " + aspl.getProbability(ctr));
+            ListIterator<SpanNode> lIter = aspl.childIterator(ctr);
+            while (lIter.hasNext()) {
+                SpanNode spnNode = lIter.next();
+                if (debug) System.out.println("Parsing span node: [" + spnNode.getFrom() + ", " + spnNode.getTo() + "] ");
+                if (spnNode instanceof AlternateSpanList) {
+                    if (debug) System.out.println("A child alternate span list found. Recursing");
+                    parseAlternateLists((AlternateSpanList)spnNode);
+                } else if (spnNode instanceof SpanList) {
+                    if (debug) System.out.println("A child span list found. Recursing");
+                    consumeAnnotations((SpanList)spnNode);
+                } else {
+                    //System.out.println("Span Node (from alternate spanlist): " + spnNode);
+                    getAnnotationsForNode(spnNode);
+                }
+            }
+            ctr ++;
+        }
+    }
+
+
+    public void parseFieldForAnnotations(StringFieldValue sfv) {
+        Collection<SpanTree> c = sfv.getSpanTrees();
+        Iterator<SpanTree> iiter = c.iterator();
+        while (iiter.hasNext()) {
+            if (debug) System.out.println(sfv + " has annotations");
+            tree = iiter.next();
+            SpanList root = (SpanList) tree.getRoot();
+            consumeAnnotations(root);
+        }
+    }
+
+
+    public void getAnnotationsForNode(SpanNode node) {
+        Iterator<Annotation> iter = tree.iterator(node);
+        boolean annotationPresent = false;
+		while (iter.hasNext()) {
+            annotationPresent = true;
+			Annotation xx = iter.next();
+			AnnotationType t = xx.getType();
+			StringFieldValue fValue = (StringFieldValue) xx.getFieldValue();
+			if (debug) System.out.println("Annotation: " + xx);
+            if (fValue == null) {
+                if (debug) System.out.println("Field Value is null");
+                return;
+            } else {
+                if (debug) System.out.println("Field Value: " + fValue.getString());
+            }
+		}
+        if (!annotationPresent) {
+            if (debug) System.out.println("****No annotations found for the span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanListTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanListTestCase.java
new file mode 100755
index 00000000000..73e2c65fd5f
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SpanListTestCase.java
@@ -0,0 +1,354 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SpanListTestCase extends AbstractTypesTest {
+
+    @Test
+    public void testSerializeDeserialize() {
+        {
+            SpanList spanList = new SpanList();
+            serializeAndAssert(spanList);
+        }
+        {
+            SpanList spanList = new SpanList();
+            Span s1 = new Span(1, 2);
+            Span s2 = new Span(3, 4);
+            Span s3 = new Span(4, 5);
+            spanList.add(s1).add(s2).add(s3);
+            SpanList s4 = new SpanList();
+            Span s5 = new Span(7, 8);
+            Span s6 = new Span(8, 9);
+            s4.add(s5).add(s6);
+            spanList.add(s4);
+
+            serializeAndAssert(spanList);
+        }
+    }
+
+    private void serializeAndAssert(SpanList spanList) {
+        GrowableByteBuffer buffer;
+        {
+            buffer = new GrowableByteBuffer(1024);
+            DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+            StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lk");
+            SpanTree tree = new SpanTree("bababa", spanList);
+            value.setSpanTree(tree);
+            serializer.write(null, value);
+            buffer.flip();
+        }
+        SpanList spanList2;
+        {
+            DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+            StringFieldValue value = new StringFieldValue();
+            deserializer.read(null, value);
+            spanList2 = (SpanList)value.getSpanTree("bababa").getRoot();
+        }
+
+        assertEquals(spanList, spanList2);
+        assertNotSame(spanList, spanList2);
+    }
+
+    @Test
+    public void testFromAndTo() {
+        Span s1 = new Span(1, 2);
+        Span s2 = new Span(5, 10);
+
+        SpanList list = new SpanList();
+
+        assertEquals(-1, list.getFrom());
+        assertEquals(-1, list.getTo());
+
+        list.add(s1);
+        assertEquals(1, list.getFrom());
+        assertEquals(1, list.getFrom());
+        assertEquals(3, list.getTo());
+        assertEquals(3, list.getTo());
+
+        list.add(s2);
+        assertEquals(1, list.getFrom());
+        assertEquals(1, list.getFrom());
+        assertEquals(15, list.getTo());
+        assertEquals(15, list.getTo());
+
+        list.clearChildren();
+        assertEquals(-1, list.getFrom());
+        assertEquals(-1, list.getTo());
+
+        s1 = new Span(1, 2);
+        s2 = new Span(5, 10);
+
+        list.add(s1);
+        assertEquals(1, list.getFrom());
+        assertEquals(1, list.getFrom());
+        assertEquals(3, list.getTo());
+        assertEquals(3, list.getTo());
+
+        list.add(s2);
+        assertEquals(1, list.getFrom());
+        assertEquals(1, list.getFrom());
+        assertEquals(15, list.getTo());
+        assertEquals(15, list.getTo());
+
+        list.remove(s1);
+        assertEquals(5, list.getFrom());
+        assertEquals(5, list.getFrom());
+        assertEquals(15, list.getTo());
+        assertEquals(15, list.getTo());
+
+        list.remove(s2);
+        assertEquals(-1, list.getFrom());
+        assertEquals(-1, list.getTo());
+    }
+
+    @Test
+    public void testSortRecursive() {
+        SpanList root = new SpanList();
+        Span a1 = new Span(0, 1);
+        SpanList b1 = new SpanList();
+        SpanList c1 = new SpanList();
+        Span d1 = new Span(9, 1);
+        root.add(d1).add(c1).add(a1).add(b1);
+
+        Span aB2 = new Span(1, 1);
+        Span bB2 = new Span(2, 1);
+        Span cB2 = new Span(3, 1);
+        Span dB2 = new Span(4, 1);
+        b1.add(dB2).add(cB2).add(bB2).add(aB2);
+
+        Span aC2 = new Span(5, 1);
+        Span bC2 = new Span(6, 1);
+        Span cC2 = new Span(7, 1);
+        Span dC2 = new Span(8, 1);
+        c1.add(cC2).add(aC2).add(bC2).add(dC2);
+
+        root.sortChildrenRecursive();
+        assertSame(a1, root.children().get(0));
+        assertSame(b1, root.children().get(1));
+        assertSame(c1, root.children().get(2));
+        assertSame(d1, root.children().get(3));
+        assertSame(aB2, b1.children().get(0));
+        assertSame(bB2, b1.children().get(1));
+        assertSame(cB2, b1.children().get(2));
+        assertSame(dB2, b1.children().get(3));
+        assertSame(aC2, c1.children().get(0));
+        assertSame(bC2, c1.children().get(1));
+        assertSame(cC2, c1.children().get(2));
+        assertSame(dC2, c1.children().get(3));
+    }
+
+    @Test
+    public void testTwoLevelFromAndTo() {
+        SpanList root = new SpanList();
+        SpanList l1 = new SpanList();
+        root.add(l1);
+
+        Span s1 = new Span(0, 20);
+        Span s2 = new Span(20, 20);
+
+        l1.add(s1).add(s2);
+
+        assertEquals(0, root.getFrom());
+        assertEquals(40, root.getTo());
+        assertEquals(0, l1.getFrom());
+        assertEquals(40, l1.getTo());
+
+        Span s3 = new Span(40, 20);
+        l1.add(s3);
+
+        assertEquals(0, root.getFrom());
+        assertEquals(60, root.getTo());
+        assertEquals(0, l1.getFrom());
+        assertEquals(60, l1.getTo());
+    }
+
+    @Test
+    public void testAddingToManyRoots() {
+        Span s1 = new Span(1, 1);
+        Span s2 = new Span(2, 1);
+        Span s3 = new Span(3, 1);
+
+        SpanList sl1 = new SpanList();
+        sl1.add(s1).add(s2).add(s3);
+
+        SpanList sl2 = new SpanList();
+        try {
+            sl2.add(s1).add(s2).add(s3);
+            fail("Should have failed here!!");
+        } catch (IllegalStateException ise) {
+            //OK!
+        }
+
+        SpanTree tree = new SpanTree("foo", sl1);
+        assertSame(tree, sl1.getParent());
+        assertSame(sl1, tree.getRoot());
+
+        SpanList sl3 = new SpanList();
+        sl1.add(sl3);
+        assertSame(sl3, sl1.children().get(3));
+        assertSame(sl1, sl3.getParent());
+
+        assertSame(tree, sl3.getSpanTree());
+        assertSame(tree, sl1.getSpanTree());
+
+        assertNull(sl2.getSpanTree());
+    }
+
+    @Test
+    public void testRemoveInvalidate() {
+        SpanList sl1 = new SpanList();
+        Span s1 = new Span(1, 2);
+
+        sl1.add(s1);
+
+        SpanList sl2 = new SpanList();
+        try {
+            sl2.add(s1);
+            fail("Should have failed.");
+        } catch (IllegalStateException ise) {
+            //OK!
+        }
+
+        sl1.remove(0);
+
+        try {
+            sl2.add(s1);
+            fail("Should have failed.");
+        } catch (IllegalStateException ise) {
+            //OK!
+        }
+    }
+
+    @Test
+    public void testMoveSimple() {
+        SpanList sl1 = new SpanList();
+        SpanList sl2 = new SpanList();
+        Span s1 = new Span(1, 2);
+
+        sl1.add(s1);
+        assertEquals(1, sl1.children().size());
+        assertEquals(0, sl2.children().size());
+        sl1.move(s1, sl2);
+        assertEquals(0, sl1.children().size());
+        assertEquals(1, sl2.children().size());
+        sl2.move(0, sl1);
+        assertEquals(1, sl1.children().size());
+        assertEquals(0, sl2.children().size());
+    }
+
+    @Test
+    public void testMoveAlternate() {
+        AlternateSpanList asl1 = new AlternateSpanList();
+        AlternateSpanList asl2 = new AlternateSpanList();
+        Span s1 = new Span(1, 2);
+
+        asl1.add(s1);
+        assertEquals(1, asl1.children().size());
+        assertEquals(0, asl2.children().size());
+        asl1.move(s1, asl2);
+        assertEquals(0, asl1.children().size());
+        assertEquals(1, asl2.children().size());
+        asl2.move(0, asl1);
+        assertEquals(1, asl1.children().size());
+        assertEquals(0, asl2.children().size());
+    }
+
+    @Test
+    public void testMoveAlternateAdvances() {
+        AlternateSpanList asl1 = new AlternateSpanList();
+        AlternateSpanList asl2 = new AlternateSpanList();
+        Span s1 = new Span(1, 2);
+
+        asl1.addChildren(new ArrayList<SpanNode>(), 50d);
+        asl1.addChildren(new ArrayList<SpanNode>(), 50d);
+        asl2.addChildren(new ArrayList<SpanNode>(), 50d);
+        asl2.addChildren(new ArrayList<SpanNode>(), 50d);
+
+        asl1.add(s1);
+        assertEquals(1, asl1.children(0).size());
+        assertEquals(0, asl1.children(1).size());
+        assertEquals(0, asl1.children(2).size());
+        assertEquals(0, asl2.children(0).size());
+        assertEquals(0, asl2.children(1).size());
+        assertEquals(0, asl2.children(2).size());
+        asl1.move(s1, asl2, 2);
+        assertEquals(0, asl1.children(0).size());
+        assertEquals(0, asl1.children(1).size());
+        assertEquals(0, asl1.children(2).size());
+        assertEquals(0, asl2.children(0).size());
+        assertEquals(0, asl2.children(1).size());
+        assertEquals(1, asl2.children(2).size());
+        asl2.move(2, 0, asl1, 1);
+        assertEquals(0, asl1.children(0).size());
+        assertEquals(1, asl1.children(1).size());
+        assertEquals(0, asl1.children(2).size());
+        assertEquals(0, asl2.children(0).size());
+        assertEquals(0, asl2.children(1).size());
+        assertEquals(0, asl2.children(2).size());
+    }
+
+    @Test
+    public void testGetStringFieldValue() {
+        StringFieldValue text = getAnnotatedString();
+        {
+            SpanTree tree = text.getSpanTree("ballooo");
+            assertSame(text, tree.getStringFieldValue());
+
+            AlternateSpanList root = (AlternateSpanList)tree.getRoot();
+            Iterator<SpanNode> it = root.childIteratorRecursive();
+
+            while (it.hasNext()) {
+                assertSame(text, it.next().getStringFieldValue());
+            }
+        }
+        {
+            SpanTree tree = text.getSpanTree("fruits");
+            assertSame(text, tree.getStringFieldValue());
+
+            SpanList root = (SpanList)tree.getRoot();
+            Iterator<SpanNode> it = root.childIteratorRecursive();
+
+            while (it.hasNext()) {
+                assertSame(text, it.next().getStringFieldValue());
+            }
+        }
+        {
+            SpanTree tree = text.removeSpanTree("ballooo");
+            assertNull(tree.getStringFieldValue());
+
+            AlternateSpanList root = (AlternateSpanList)tree.getRoot();
+            Iterator<SpanNode> it = root.childIteratorRecursive();
+
+            while (it.hasNext()) {
+                assertNull(it.next().getStringFieldValue());
+            }
+        }
+        {
+            SpanTree tree = text.removeSpanTree("fruits");
+            assertNull(tree.getStringFieldValue());
+
+            SpanList root = (SpanList)tree.getRoot();
+            Iterator<SpanNode> it = root.childIteratorRecursive();
+
+            while (it.hasNext()) {
+                assertNull(it.next().getStringFieldValue());
+            }
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanNodeAdvTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanNodeAdvTestCase.java
new file mode 100644
index 00000000000..9277d939168
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SpanNodeAdvTestCase.java
@@ -0,0 +1,333 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.After;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:mpraveen@yahoo-inc.com">Praveen Mohan</a>
+ *
+ * This test covers all possible scenarios in SpanNode.
+ */
+
+
+
+public class SpanNodeAdvTestCase {
+
+    private boolean debug = false;
+
+    private AnnotationType at1 = new AnnotationType("person", DataType.STRING);
+    private AnnotationType at2 = new AnnotationType("street", DataType.STRING);
+    private AnnotationType at3 = new AnnotationType("city", DataType.STRING);
+
+    public SpanTree tree = null;
+
+    private SpanNode span1, span2, span3;
+    private SpanNode span11, span22, span33;
+    private SpanNode span111, span222, span333;
+
+    private AlternateSpanList root, alternate1, alternate2;
+    private SpanList branch, branch1;
+    private List<SpanNode> subtreeList1, subtreeList2, subtreeList3, subtreeList4;
+    private Annotation an111;
+
+
+    public void populateSpanTree() {
+
+        root = new AlternateSpanList();
+        tree = new SpanTree("test", root);
+
+        span1 = new Span(10, 8);
+        span2 = new Span(5, 10);
+        span3 = new Span(13, 2);
+
+        span11 = new Span(0, 2);
+        span22 = new Span(2, 10);
+        span33 = new Span(8, 10);
+
+        span111 = new Span(5, 10);
+        span222 = new Span(10, 10);
+        span333 = new Span(20, 10);
+
+        an111 = new Annotation(at1);
+
+        tree.annotate(span1, at1).annotate(span2, at2).annotate(span3, at3);
+        tree.annotate(span11, at1).annotate(span22, at2).annotate(span33, at3);
+        tree.annotate(span111, an111).annotate(span222, at2).annotate(span333, at3).annotate(span333, at1);
+        tree.annotate(span222, at2);
+
+        root.add(span3);
+        root.add(span2);
+        root.add(span1);
+
+        alternate1 = new AlternateSpanList();
+        alternate1.add(span11);
+        branch = new SpanList();
+        branch.add(span22);
+        subtreeList1 = new ArrayList<SpanNode>();
+        subtreeList1.add(branch);
+        alternate1.addChildren(1, subtreeList1, 50.0d);
+
+        alternate2 = new AlternateSpanList();
+        alternate2.add(span33);
+        subtreeList2 = new ArrayList<SpanNode>();
+        subtreeList2.add(span111);
+        subtreeList2.add(span222);
+        alternate2.addChildren(1, subtreeList2, 70.0d);
+
+        subtreeList3 = new ArrayList<SpanNode>();
+        branch1 = new SpanList();
+        branch1.add(span333);
+        subtreeList3.add(branch1);
+        alternate2.addChildren(2, subtreeList3, 90.0d);
+        branch.add(alternate2);
+
+        subtreeList4 = new ArrayList<SpanNode>();
+        subtreeList4.add(alternate1);
+        root.addChildren(1, subtreeList4, 10.0d);
+    }
+
+	@Test
+	public void assertIsValid() {
+		populateSpanTree();
+		root.clearChildren(1);
+		root.remove(span3);
+
+        assertTrue(root.isValid());
+        assertTrue(span1.isValid());
+        assertTrue(span2.isValid());
+
+        assertFalse(span3.isValid());
+		assertFalse(alternate1.isValid());
+        assertFalse(span11.isValid());
+		assertFalse(branch.isValid());
+		assertFalse(span22.isValid());
+		assertFalse(alternate2.isValid());
+		assertFalse(span33.isValid());
+		assertFalse(branch1.isValid());
+		assertFalse(span333.isValid());
+		assertFalse(span111.isValid());
+		assertFalse(span222.isValid());
+	}
+
+    @Test
+	public void assertIsLeafNode() {
+		populateSpanTree();
+		assertFalse(root.isLeafNode());
+		assertFalse(alternate1.isLeafNode());
+		assertFalse(branch.isLeafNode());
+		assertTrue(span11.isLeafNode());
+		assertTrue(span22.isLeafNode());
+		assertFalse(alternate2.isLeafNode());
+		assertTrue(span33.isLeafNode());
+		assertFalse(branch1.isLeafNode());
+		assertTrue(span111.isLeafNode());
+		assertTrue(span222.isLeafNode());
+		assertTrue(span333.isLeafNode());
+	}
+
+	@Test
+	public void assertOverlaps() {
+		populateSpanTree();
+		assertTrue(span1.overlaps(span2));
+		assertTrue(span1.overlaps(span3));
+		assertTrue(span2.overlaps(span3));
+		assertFalse(span11.overlaps(span22));
+		assertTrue(span22.overlaps(span33));
+		assertFalse(span11.overlaps(span33));
+		assertTrue(span111.overlaps(span222));
+		assertFalse(span111.overlaps(span333));
+		assertFalse(span222.overlaps(span333));
+		assertTrue(span1.overlaps(span222));
+		assertFalse(span1.overlaps(span333));
+		assertTrue(span2.overlaps(span222));
+		assertFalse(span3.overlaps(span22));
+
+		assertTrue(span2.overlaps(span1));
+		assertTrue(span3.overlaps(span1));
+		assertTrue(span3.overlaps(span2));
+		assertFalse(span22.overlaps(span11));
+		assertTrue(span33.overlaps(span22));
+		assertFalse(span33.overlaps(span11));
+		assertTrue(span222.overlaps(span111));
+		assertFalse(span333.overlaps(span111));
+		assertFalse(span333.overlaps(span222));
+		assertTrue(span222.overlaps(span1));
+		assertFalse(span333.overlaps(span1));
+		assertTrue(span222.overlaps(span2));
+		assertTrue(span1.overlaps(span1));
+		assertFalse(span22.overlaps(span3));
+		assertFalse(root.overlaps(alternate1));
+		assertTrue(alternate2.overlaps(span22));
+		assertTrue(branch.overlaps(root));
+		assertTrue(branch.overlaps(alternate2));
+		assertTrue(root.overlaps(alternate2));
+		assertTrue(span22.overlaps(root));
+
+	}
+
+	@Test
+	public void assertContains() {
+		populateSpanTree();
+		assertTrue(span222.contains(span1));
+		assertFalse(span1.contains(span222));
+		assertTrue(span1.contains(span3));
+		assertFalse(span33.contains(span111));
+		assertTrue(span222.contains(span3));
+		assertTrue(span111.contains(span2));
+		assertTrue(span2.contains(span111));
+		assertTrue(branch.contains(root));
+		assertTrue(branch.contains(alternate2));
+		assertTrue(root.contains(alternate2));
+		assertFalse(alternate2.contains(span22));
+	}
+
+    @Test
+	public void assertCompareTo() {
+		populateSpanTree();
+		assertEquals(1 , span1.compareTo(span2));
+		assertEquals(-1, span2.compareTo(span1));
+		assertEquals(-1 , span2.compareTo(span3));
+		assertEquals(1, span3.compareTo(span2));
+		assertEquals(0, span2.compareTo(span111));
+		assertEquals(1, root.compareTo(branch));
+		assertEquals(-1, alternate1.compareTo(root));
+		assertEquals(1, branch.compareTo(span22));
+		assertEquals(-1, branch.compareTo(alternate2));
+		assertEquals(1, alternate2.compareTo(root));
+		assertEquals(-1, span111.compareTo(root));
+		assertEquals(0, span333.compareTo(branch1));
+		assertEquals(0, alternate2.compareTo(span33));
+		root.removeChildren();
+		tree.cleanup();
+		assertEquals(1, span11.compareTo(root));
+	}
+
+    @Test
+	public void assertGetParent() {
+		populateSpanTree();
+		assertEquals(root, span1.getParent());
+		assertEquals(root, span2.getParent());
+        assertEquals(root, span3.getParent());
+		assertEquals(root, alternate1.getParent());
+		assertEquals(alternate1, span11.getParent());
+		assertEquals(alternate1, branch.getParent());
+		assertEquals(branch, span22.getParent());
+		assertEquals(branch, alternate2.getParent());
+		assertEquals(alternate2, span33.getParent());
+		assertEquals(alternate2, span111.getParent());
+		assertEquals(branch1, span333.getParent());
+		assertEquals(alternate2, branch1.getParent());
+		assertEquals(alternate1, span11.getParent());
+	}
+
+
+    @After
+    public void tearDown() {
+        tree = null;
+    }
+
+
+    public void consumeAnnotations(SpanList root) {
+        if (root instanceof AlternateSpanList) {
+            parseAlternateLists((AlternateSpanList)root);
+            if (debug) System.out.println("\nGetting annotations for the SpanList itself : [" + root.getFrom() + ", " + root.getTo() + "] ");
+            getAnnotationsForNode(root);
+            return;
+        }
+        if (debug) System.out.println("\n\nSpanList: [" + root.getFrom() + ", " + root.getTo() + "] num Children: " + root.numChildren());
+        if (debug) System.out.println("-------------------");
+        Iterator<SpanNode> childIterator = root.childIterator();
+		while (childIterator.hasNext()) {
+			SpanNode node = childIterator.next();
+            //System.out.println("Span Node: " + node); // + " Span Text: " + node.getText(fieldValStr));
+            if (debug) System.out.println("\n\nSpan Node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+            if (node instanceof AlternateSpanList) {
+                parseAlternateLists((AlternateSpanList)node);
+                if (debug) System.out.println("---- Alternate SpanList complete ---");
+            } else if (node instanceof SpanList) {
+                if (debug) System.out.println("Encountered another span list");
+                SpanList spl = (SpanList) node;
+                ListIterator<SpanNode> lli = spl.childIterator();
+                while (lli.hasNext()) System.out.print(" " + lli.next() + " ");
+                consumeAnnotations((SpanList) node);
+            } else {
+       			if (debug) System.out.println("\nGetting annotations for this span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+   	    	    getAnnotationsForNode(node);
+            }
+		}
+        if (debug) System.out.println("\nGetting annotations for the SpanList itself : [" + root.getFrom() + ", " + root.getTo() + "] ");
+        getAnnotationsForNode(root);
+    }
+
+    public void parseAlternateLists(AlternateSpanList aspl) {
+        int no = aspl.getNumSubTrees();
+        if (debug) System.out.println("Parsing Alternate span list. No of subtrees: " + no);
+        int ctr = 0;
+        while (ctr < no) {
+            if (debug) System.out.println("\nSubTree: " + ctr + " probability: " + aspl.getProbability(ctr));
+            ListIterator<SpanNode> lIter = aspl.childIterator(ctr);
+            while (lIter.hasNext()) {
+                SpanNode spnNode = lIter.next();
+                if (debug) System.out.println("Parsing span node: [" + spnNode.getFrom() + ", " + spnNode.getTo() + "] ");
+                if (spnNode instanceof AlternateSpanList) {
+                    if (debug) System.out.println("A child alternate span list found. Recursing");
+                    parseAlternateLists((AlternateSpanList)spnNode);
+                } else if (spnNode instanceof SpanList) {
+                    if (debug) System.out.println("A child span list found. Recursing");
+                    consumeAnnotations((SpanList)spnNode);
+                } else {
+                    //System.out.println("Span Node (from alternate spanlist): " + spnNode);
+                    getAnnotationsForNode(spnNode);
+                }
+            }
+            ctr ++;
+        }
+    }
+
+
+    public void parseFieldForAnnotations(StringFieldValue sfv) {
+        Collection<SpanTree> c = sfv.getSpanTrees();
+        Iterator<SpanTree> iiter = c.iterator();
+        while (iiter.hasNext()) {
+            if (debug) System.out.println(sfv + " has annotations");
+            tree = iiter.next();
+            SpanList root = (SpanList) tree.getRoot();
+            consumeAnnotations(root);
+        }
+    }
+
+
+    public void getAnnotationsForNode(SpanNode node) {
+        Iterator<Annotation> iter = tree.iterator(node);
+        boolean annotationPresent = false;
+		while (iter.hasNext()) {
+            annotationPresent = true;
+			Annotation xx = iter.next();
+			AnnotationType t = xx.getType();
+			StringFieldValue fValue = (StringFieldValue) xx.getFieldValue();
+			if (debug) System.out.println("Annotation: " + xx);
+            if (fValue == null) {
+                if (debug) System.out.println("Field Value is null");
+                return;
+            } else {
+                if (debug) System.out.println("Field Value: " + fValue.getString());
+            }
+		}
+        if (!annotationPresent) {
+            if (debug) System.out.println("****No annotations found for the span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanNodeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanNodeTestCase.java
new file mode 100644
index 00000000000..b7d17779ea9
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SpanNodeTestCase.java
@@ -0,0 +1,646 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SpanNodeTestCase extends junit.framework.TestCase {
+
+    public void testOverlaps() {
+        //qwertyuiopasdfghjklzxcvbnm
+        //012345678901234567890123456789
+        Span a = new Span(0, 6);  //qwerty
+        Span b = new Span(6, 6);  //uiopas
+        Span c = new Span(12, 5); //dfghj
+        Span d = new Span(17, 6); //klzxcv
+        Span e = new Span(23, 3); //bnm
+
+        assertFalse(a.overlaps(b));
+        assertFalse(b.overlaps(a));
+
+        assertFalse(b.overlaps(c));
+        assertFalse(c.overlaps(b));
+
+        assertFalse(c.overlaps(d));
+        assertFalse(d.overlaps(c));
+
+        assertFalse(d.overlaps(e));
+        assertFalse(e.overlaps(d));
+
+
+        Span all = new Span(0, 25);
+
+        assertTrue(a.overlaps(all));
+        assertTrue(all.overlaps(a));
+
+        assertTrue(b.overlaps(all));
+        assertTrue(all.overlaps(b));
+
+        assertTrue(c.overlaps(all));
+        assertTrue(all.overlaps(c));
+
+        assertTrue(all.overlaps(d));
+        assertTrue(d.overlaps(all));
+
+        assertTrue(all.overlaps(e));
+        assertTrue(e.overlaps(all));
+
+
+        Span f = new Span(3, 7); // rtyuiop
+
+        assertTrue(a.overlaps(f));
+        assertTrue(f.overlaps(a));
+
+        assertTrue(b.overlaps(f));
+        assertTrue(f.overlaps(b));
+
+        assertFalse(c.overlaps(f));
+        assertFalse(f.overlaps(c));
+
+        assertFalse(d.overlaps(f));
+        assertFalse(f.overlaps(d));
+
+        assertFalse(e.overlaps(f));
+        assertFalse(f.overlaps(e));
+
+        assertTrue(all.overlaps(f));
+        assertTrue(f.overlaps(all));
+
+
+        Span g = new Span(0, 7);  //qwertyu
+
+        assertTrue(a.overlaps(g));
+        assertTrue(g.overlaps(a));
+
+        assertTrue(b.overlaps(g));
+        assertTrue(g.overlaps(b));
+
+        assertFalse(c.overlaps(g));
+        assertFalse(g.overlaps(c));
+
+        assertFalse(d.overlaps(g));
+        assertFalse(g.overlaps(d));
+
+        assertFalse(e.overlaps(g));
+        assertFalse(g.overlaps(e));
+
+        assertTrue(f.overlaps(g));
+        assertTrue(g.overlaps(f));
+
+        assertTrue(all.overlaps(g));
+        assertTrue(g.overlaps(all));
+
+
+        assertTrue(a.overlaps(a));
+    }
+
+    public void testContains() {
+        //qwertyuiopasdfghjklzxcvbnm
+        //012345678901234567890123456789
+        Span a = new Span(0, 6);  //qwerty
+        Span b = new Span(6, 6);  //uiopas
+        Span all = new Span(0, 25);
+
+        assertFalse(a.contains(all));
+        assertTrue(all.contains(a));
+
+        assertFalse(a.contains(b));
+        assertFalse(b.contains(a));
+
+        assertTrue(a.contains(a));
+
+        Span c = new Span(0, 7);  //qwertyu
+
+        assertFalse(a.contains(c));
+        assertTrue(c.contains(a));
+    }
+
+
+    public void testSpanTree() {
+        final String text = "Hallo er et ord fra Norge";
+        SpanList sentence = new SpanList();
+        SpanTree tree = new SpanTree("sentence", sentence);
+
+        //Locate words and whitespace:
+        SpanNode hallo = new Span(0, 5);
+        SpanNode spc1 = new Span(5, 1);
+        SpanNode er = new Span(6, 2);
+        SpanNode spc2 = new Span(8, 1);
+        SpanNode et = new Span(9, 2);
+        SpanNode spc3 = new Span(11, 1);
+        SpanNode ord = new Span(12, 3);
+        SpanNode spc4 = new Span(15, 1);
+        SpanNode fra = new Span(16, 3);
+        SpanNode spc5 = new Span(19, 1);
+        SpanNode norge = new Span(20, 5);
+
+        AnnotationType noun = new AnnotationType("noun");
+        AnnotationType verb = new AnnotationType("verb");
+        AnnotationType prep = new AnnotationType("preposition");
+        AnnotationType det = new AnnotationType("determiner");
+        AnnotationType noun_phrase = new AnnotationType("noun_phrase");
+        AnnotationType verb_phrase = new AnnotationType("verb_phrase");
+        AnnotationType prep_phrase = new AnnotationType("preposition_phrase");
+        AnnotationType separator = new AnnotationType("separator");
+        AnnotationType sentenceType = new AnnotationType("sentence");
+
+        //Determine word classes and add annotation for them:
+        tree.annotate(hallo, noun);
+        tree.annotate(spc1, separator);
+        tree.annotate(er, verb);
+        tree.annotate(spc2, separator);
+        tree.annotate(et, det);
+        tree.annotate(spc3, separator);
+        tree.annotate(ord, noun);
+        tree.annotate(spc4, separator);
+        tree.annotate(fra, prep);
+        tree.annotate(spc5, separator);
+        tree.annotate(norge, noun);
+
+
+        //Identify phrases and build natural language parse tree, and annotate as we go:
+        tree.annotate(hallo, noun_phrase);
+
+        SpanList np2 = new SpanList();
+        np2.children().add(et);
+        np2.children().add(spc3);
+        np2.children().add(ord);
+        tree.annotate(np2, noun_phrase);
+
+        tree.annotate(norge, noun_phrase);
+
+        SpanList pp = new SpanList();
+        pp.children().add(fra);
+        pp.children().add(spc5);
+        pp.children().add(norge);
+        tree.annotate(pp, prep_phrase);
+
+        SpanList np3 = new SpanList();
+        np3.children().add(np2);
+        np3.children().add(spc4);
+        np3.children().add(pp);
+        tree.annotate(np3, noun_phrase);
+
+        SpanList vp = new SpanList();
+        vp.children().add(er);
+        vp.children().add(spc2);
+        vp.children().add(np3);
+        tree.annotate(vp, verb_phrase);
+
+        sentence.children().add(hallo);
+        sentence.children().add(spc1);
+        sentence.children().add(vp);
+        tree.annotate(sentence, sentenceType);
+
+
+        //assert that extracted text is correct:
+        assertEquals("Hallo", hallo.getText(text));
+        assertEquals(" ", spc1.getText(text));
+        assertEquals("er", er.getText(text));
+        assertEquals(" ", spc2.getText(text));
+        assertEquals("et", et.getText(text));
+        assertEquals(" ", spc3.getText(text));
+        assertEquals("ord", ord.getText(text));
+        assertEquals(" ", spc4.getText(text));
+        assertEquals("fra", fra.getText(text));
+        assertEquals(" ", spc5.getText(text));
+        assertEquals("Norge", norge.getText(text));
+
+        assertEquals("et ord", np2.getText(text).toString());
+        assertEquals("fra Norge", pp.getText(text).toString());
+        assertEquals("et ord fra Norge", np3.getText(text).toString());
+        assertEquals("er et ord fra Norge", vp.getText(text).toString());
+        assertEquals("Hallo er et ord fra Norge", sentence.getText(text).toString());
+
+        //coming up: assert that children(Annotation) works...
+    }
+
+    public void testOrder() {
+        {
+            String text = "08/20/1999";
+                         //012345678901
+            Span d = new Span(3, 2);
+            Span m = new Span(0, 2);
+            Span y = new Span(6, 4);
+
+            SpanList date = new SpanList();
+            date.children().add(d);
+            date.children().add(m);
+            date.children().add(y);
+
+            assertEquals("20081999", date.getText(text).toString());
+
+            assertEquals(0, date.getFrom());
+            assertEquals(10, date.getTo());
+            assertEquals(10, date.getLength());
+        }
+        {
+            String text = "20/08/2000";
+            //012345678901
+            Span d = new Span(0, 2);
+            Span m = new Span(3, 2);
+            Span y = new Span(6, 4);
+
+            SpanList date = new SpanList();
+            date.children().add(d);
+            date.children().add(m);
+            date.children().add(y);
+
+            assertEquals("20082000", date.getText(text).toString());
+
+            assertEquals(0, date.getFrom());
+            assertEquals(10, date.getTo());
+            assertEquals(10, date.getLength());
+        }
+    }
+
+    public void testNonRecursiveAnnotationIterator() {
+        AnnotationType nounType = new AnnotationType("noun");
+        AnnotationType detType = new AnnotationType("determiner");
+        AnnotationType wordType = new AnnotationType("word");
+        AnnotationType begTagType = new AnnotationType("begin_tag");
+        AnnotationType cmpWordType = new AnnotationType("compound_word");
+
+        SpanNode span = new Span(0,2);
+        SpanTree tree = new SpanTree("span", span);
+
+        Annotation word = new Annotation(wordType);
+        Annotation bgtg = new Annotation(begTagType);
+        Annotation cpwd = new Annotation(cmpWordType);
+        Annotation detr = new Annotation(detType);
+        Annotation noun = new Annotation(nounType);
+
+        tree.annotate(span, word);
+        tree.annotate(span, bgtg);
+        tree.annotate(span, cpwd);
+        tree.annotate(span, detr);
+        tree.annotate(span, noun);
+
+
+        {
+            Iterator<Annotation> it = tree.iterator();
+            assertSame(word, it.next());
+            assertSame(bgtg, it.next());
+            assertSame(cpwd, it.next());
+            assertSame(detr, it.next());
+            assertSame(noun, it.next());
+            try {
+                it.next();
+                fail("Should have gotten NoSuchElementException");
+            } catch (NoSuchElementException e) {
+                //ignore
+            }
+        }
+        {
+            Iterator<Annotation> it = tree.iterator();
+            for (int i = 0; i < 100; i++) {
+                assertTrue(it.hasNext());
+            }
+        }
+        {
+            Iterator<Annotation> it = tree.iterator();
+            assertTrue(it.hasNext());
+            assertSame(word, it.next());
+            assertTrue(it.hasNext());
+            assertSame(bgtg, it.next());
+            assertTrue(it.hasNext());
+            assertSame(cpwd, it.next());
+            assertTrue(it.hasNext());
+            assertSame(detr, it.next());
+            assertTrue(it.hasNext());
+            assertSame(noun, it.next());
+            assertFalse(it.hasNext());
+        }
+
+
+
+        {
+            Iterator<Annotation> it = tree.iterator();
+            assertSame(word, it.next());
+            assertSame(bgtg, it.next());
+            assertSame(cpwd, it.next());
+            assertSame(detr, it.next());
+            assertSame(noun, it.next());
+            try {
+                it.next();
+                fail("Should have gotten NoSuchElementException");
+            } catch (NoSuchElementException e) {
+                //ignore
+            }
+        }
+        {
+            Iterator<Annotation> it = tree.iterator();
+            for (int i = 0; i < 100; i++) {
+                assertTrue(it.hasNext());
+            }
+        }
+        {
+            Iterator<Annotation> it = tree.iterator();
+            assertTrue(it.hasNext());
+            assertSame(word, it.next());
+            assertTrue(it.hasNext());
+            assertSame(bgtg, it.next());
+            assertTrue(it.hasNext());
+            assertSame(cpwd, it.next());
+            assertTrue(it.hasNext());
+            assertSame(detr, it.next());
+            assertTrue(it.hasNext());
+            assertSame(noun, it.next());
+            assertFalse(it.hasNext());
+        }
+
+    }
+
+    public void testRecursion() {
+        AnnotationType noun = new AnnotationType("noun");
+        AnnotationType verb = new AnnotationType("verb");
+        AnnotationType sentenceType = new AnnotationType("sentence");
+        AnnotationType word = new AnnotationType("word");
+        AnnotationType phraseType = new AnnotationType("phrase");
+
+        String text = "There is no bizniz like showbizniz";
+                     //0123456789012345678901234567890123456789
+        SpanList sentence = new SpanList();
+        SpanTree tree = new SpanTree("sentence", sentence);
+
+        Span there = new Span(0, 5);
+        Span is = new Span(6, 2);
+        Span no = new Span(9, 2);
+        Span bizniz = new Span(12, 6);
+        Span like = new Span(19, 4);
+        Span showbizniz = new Span(24, 10);
+
+        tree.annotate(there, word);
+        tree.annotate(is, word);
+        tree.annotate(is, verb);
+        tree.annotate(no, word);
+        tree.annotate(bizniz, word);
+        tree.annotate(bizniz, noun);
+        tree.annotate(like, word);
+        tree.annotate(showbizniz, word);
+        tree.annotate(showbizniz, noun);
+
+        SpanList endPhrase = new SpanList();
+        endPhrase.add(like);
+        endPhrase.add(showbizniz);
+        tree.annotate(endPhrase, phraseType);
+
+        SpanList phrase = new SpanList();
+        phrase.children().add(no);
+        phrase.children().add(bizniz);
+        phrase.children().add(endPhrase);
+        tree.annotate(phrase, phraseType);
+
+        sentence.children().add(there);
+        sentence.children().add(is);
+        sentence.children().add(phrase);
+        tree.annotate(sentence, sentenceType);
+
+        Iterator<SpanNode> children;
+
+        children = sentence.childIteratorRecursive();
+
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("There", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(1, a.size());
+            assertEquals(a.get(0).getType(), word);
+        }
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("is", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(2, a.size());
+            assertEquals(a.get(0).getType(), word);
+            assertEquals(a.get(1).getType(), verb);
+        }
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("no", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(1, a.size());
+            assertEquals(a.get(0).getType(), word);
+        }
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("bizniz", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(2, a.size());
+            assertEquals(a.get(0).getType(), word);
+            assertEquals(a.get(1).getType(), noun);
+        }
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("like", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(1, a.size());
+            assertEquals(a.get(0).getType(), word);
+        }
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("showbizniz", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(2, a.size());
+            assertEquals(a.get(0).getType(), word);
+            assertEquals(a.get(1).getType(), noun);
+        }
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("likeshowbizniz", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(1, a.size());
+            assertEquals(a.get(0).getType(), phraseType);
+        }
+        {
+            assertTrue(children.hasNext());
+            SpanNode next = children.next();
+            assertEquals("nobiznizlikeshowbizniz", next.getText(text).toString());
+            List<Annotation> a = annotations(tree, next);
+            assertEquals(1, a.size());
+            assertEquals(a.get(0).getType(), phraseType);
+        }
+        {
+            assertFalse(children.hasNext());
+        }
+
+        List<Annotation> annotationList = new LinkedList<Annotation>();
+        for (Annotation a : tree) {
+            annotationList.add(a);
+        }
+        Collections.sort(annotationList);
+
+        Iterator<Annotation> annotations = annotationList.iterator();
+
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("There", annotation.getSpanNode().getText(text));
+        assertEquals(word, annotation.getType());  //there: word
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("Thereisnobiznizlikeshowbizniz", annotation.getSpanNode().getText(text).toString());
+        assertEquals(sentenceType, annotation.getType());  //sentence
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("is", annotation.getSpanNode().getText(text));
+        assertEquals(word, annotation.getType());  //is: word
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("is", annotation.getSpanNode().getText(text));
+        assertEquals(verb, annotation.getType());  //is: verb
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("no", annotation.getSpanNode().getText(text));
+        assertEquals(word, annotation.getType());  //no: word
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("nobiznizlikeshowbizniz", annotation.getSpanNode().getText(text).toString());
+        assertEquals(phraseType, annotation.getType());  //no bizniz like showbizniz: phrase
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("bizniz", annotation.getSpanNode().getText(text));
+        assertEquals(word, annotation.getType());  //bizniz: word
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("bizniz", annotation.getSpanNode().getText(text));
+        assertEquals(noun, annotation.getType());  //bizniz: noun
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("like", annotation.getSpanNode().getText(text));
+        assertEquals(word, annotation.getType());  //like: word
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("likeshowbizniz", annotation.getSpanNode().getText(text).toString());
+        assertEquals(phraseType, annotation.getType());  //like showbizniz: phrase
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("showbizniz", annotation.getSpanNode().getText(text));
+        assertEquals(word, annotation.getType());  //showbizniz: word
+        }
+        {
+        assertTrue(annotations.hasNext());
+        Annotation annotation = annotations.next();
+        assertEquals("showbizniz", annotation.getSpanNode().getText(text));
+        assertEquals(noun, annotation.getType());  //showbizniz: noun
+        }
+
+        assertFalse(annotations.hasNext());
+    }
+
+    private static List<Annotation> annotations(SpanTree tree, SpanNode node) {
+        List<Annotation> list = new ArrayList<Annotation>();
+        Iterator<Annotation> it = tree.iterator(node);
+        while (it.hasNext()) {
+            list.add(it.next());
+        }
+        return list;
+    }
+
+	public void testMultilevelRecursion() {
+		//			   01234567890123
+		String text = "Hello!Goodbye!";
+        AnnotationType block = new AnnotationType("block");
+		SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("root", root);
+
+		SpanList block1 = new SpanList();
+		SpanNode hello = new Span(0,6);
+        tree.annotate(hello, block);
+		block1.add(hello);
+
+		SpanList block2 = new SpanList();
+		SpanNode goodbye = new Span(6,8);
+        tree.annotate(goodbye, block);
+		block2.add(goodbye);
+
+		root.add(block1).add(block2);
+
+		Iterator<SpanNode> nodeIterator = root.childIteratorRecursive();
+		assertTrue(nodeIterator.hasNext());
+		assertTrue(nodeIterator.next().equals(hello));
+		assertTrue(nodeIterator.hasNext());
+		assertTrue(nodeIterator.next().equals(block1));
+		assertTrue(nodeIterator.hasNext());
+		assertTrue(nodeIterator.next().equals(goodbye));
+		assertTrue(nodeIterator.hasNext());
+		assertTrue(nodeIterator.next().equals(block2));
+		assertFalse(nodeIterator.hasNext());
+		assertTrue(root.getText(text).toString().equals(text));
+	}
+
+    public void testRecursiveIteratorDeterministicBehavior() {
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("tree", root);
+
+        SpanList a = new SpanList();
+        Span b = new Span(0,1);
+        Span c = new Span(1,1);
+        a.add(b).add(c);
+        root.add(a);
+        Span d = new Span(2,1);
+        root.add(d);
+
+        Span newC = new Span(3,1);
+
+        ListIterator<SpanNode> children = root.childIteratorRecursive();
+        assertTrue(children.hasNext());
+        assertSame(b, children.next());
+        assertTrue(children.hasNext());
+        assertSame(c, children.next());
+        assertTrue(children.hasNext());
+        assertTrue(children.hasNext());
+        assertTrue(children.hasNext());
+
+        children.set(newC);
+        assertTrue(children.hasNext());
+        assertSame(a, children.next());
+        assertTrue(children.hasNext());
+        assertSame(d, children.next());
+        assertFalse(children.hasNext());
+
+
+        assertSame(a, root.children().get(0));
+        assertSame(d, root.children().get(1));
+        assertEquals(2, root.children().size());
+
+        assertSame(b, a.children().get(0));
+        assertSame(newC, a.children().get(1));
+        assertEquals(2, a.children().size());
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanTestCase.java
new file mode 100755
index 00000000000..47df445eff1
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SpanTestCase.java
@@ -0,0 +1,100 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SpanTestCase extends AbstractTypesTest {
+
+    @Test
+    public void testIteration() {
+        Span span = new Span(1, 2);
+        ListIterator<SpanNode> b = span.childIterator();
+        assertFalse(b.hasNext());
+        assertFalse(b.hasPrevious());
+        assertEquals(0, b.nextIndex());
+        assertEquals(0, b.previousIndex());
+        try {
+            b.next();
+            fail();
+        } catch (NoSuchElementException nsee) {
+            //ok
+        }
+        try {
+            b.previous();
+            fail();
+        } catch (NoSuchElementException nsee) {
+            //ok
+        }
+        try {
+            b.remove();
+            fail();
+        } catch (UnsupportedOperationException uoe) {
+            //ok
+        }
+        try {
+            b.set(new Span(1, 1));
+            fail();
+        } catch (UnsupportedOperationException uoe) {
+            //ok
+        }
+        try {
+            b.add(new Span(1, 1));
+            fail();
+        } catch (UnsupportedOperationException uoe) {
+            //ok
+        }
+    }
+
+    @Test
+    public void testSerializeDeserialize() {
+        {
+            Span span = new Span(1, 2);
+            serializeAndAssert(span);
+        }
+        {
+            Span span = new Span(4, 15);
+            serializeAndAssert(span);
+        }
+        {
+            Span span = new Span(1, 19);
+            serializeAndAssert(span);
+        }
+    }
+
+    private void serializeAndAssert(Span span) {
+        GrowableByteBuffer buffer;
+        {
+            buffer = new GrowableByteBuffer(1024);
+            DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+            StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lk");
+            SpanTree tree = new SpanTree("bababa", span);
+            value.setSpanTree(tree);
+            serializer.write(null, value);
+            buffer.flip();
+        }
+        Span span2;
+        {
+            DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+            StringFieldValue value = new StringFieldValue();
+            deserializer.read(null, value);
+            span2 = (Span)value.getSpanTree("bababa").getRoot();
+        }
+
+        assertEquals(span, span2);
+        assertNotSame(span, span2);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanTreeAdvTest.java b/document/src/test/java/com/yahoo/document/annotation/SpanTreeAdvTest.java
new file mode 100644
index 00000000000..cfb87ee997d
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SpanTreeAdvTest.java
@@ -0,0 +1,321 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.After;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author <a href="mailto:mpraveen@yahoo-inc.com">Praveen Mohan</a>
+ *
+ * This test covers all possible scenarios in SpanTree.
+ *
+ */
+
+
+
+public class SpanTreeAdvTest {
+
+    private boolean debug = false;
+
+    private AnnotationType at1 = new AnnotationType("person", DataType.STRING);
+    private AnnotationType at2 = new AnnotationType("street", DataType.STRING);
+    private AnnotationType at3 = new AnnotationType("city", DataType.STRING);
+
+    public SpanTree tree = null;
+
+    private SpanNode span1, span2, span3;
+    private SpanNode span11, span22, span33;
+    private SpanNode span111, span222, span333;
+
+    private AlternateSpanList root, alternate1, alternate2;
+    private SpanList branch, branch1;
+    private List<SpanNode> subtreeList1, subtreeList2, subtreeList3, subtreeList4;
+    private Annotation an111;
+
+
+    public void populateSpanTree() {
+
+        root = new AlternateSpanList();
+        tree = new SpanTree("test", root);
+
+        span1 = new Span(10, 3);
+        span2 = new Span(4, 6);
+        span3 = new Span(13, 10);
+
+        span11 = new Span(0, 2);
+        span22 = new Span(2, 10);
+        span33 = new Span(12, 10);
+
+        span111 = new Span(5, 10);
+        span222 = new Span(15, 5);
+        span333 = new Span(20, 10);
+
+        an111 = new Annotation(at1);
+
+        tree.annotate(span1, at1).annotate(span2, at2).annotate(span3, at3);
+        tree.annotate(span11, at1).annotate(span22, at2).annotate(span33, at3);
+        tree.annotate(span111, an111).annotate(span222, at2).annotate(span333, at3).annotate(span333, at1);
+        tree.annotate(span222, at2);
+
+        root.add(span3);
+        root.add(span2);
+        root.add(span1);
+
+        alternate1 = new AlternateSpanList();
+        alternate1.add(span11);
+        branch = new SpanList();
+        branch.add(span22);
+        subtreeList1 = new ArrayList<SpanNode>();
+        subtreeList1.add(branch);
+        alternate1.addChildren(1, subtreeList1, 50.0d);
+
+        alternate2 = new AlternateSpanList();
+        alternate2.add(span33);
+        subtreeList2 = new ArrayList<SpanNode>();
+        subtreeList2.add(span111);
+        subtreeList2.add(span222);
+        alternate2.addChildren(1, subtreeList2, 70.0d);
+
+        subtreeList3 = new ArrayList<SpanNode>();
+        branch1 = new SpanList();
+        branch1.add(span333);
+        subtreeList3.add(branch1);
+        alternate2.addChildren(2, subtreeList3, 90.0d);
+        branch.add(alternate2);
+
+        subtreeList4 = new ArrayList<SpanNode>();
+        subtreeList4.add(alternate1);
+        root.addChildren(1, subtreeList4, 10.0d);
+    }
+
+    @Test (expected = IllegalStateException.class)
+    public void assertSharingAnnotationInstance() {
+        populateSpanTree();
+        tree.annotate(span333, an111);
+    }
+
+
+
+    @Test
+    public void assertRemoveChildrenCleanupTest() {
+        populateSpanTree();
+        root.removeChildren();
+        tree.cleanup();
+        Iterator<Annotation> iter =tree.iterator();
+        assertFalse(iter.hasNext());
+        assertEquals(0, tree.numAnnotations());
+    }
+
+
+    @Test
+    public void assertClearChildrenCleanupTest() {
+        populateSpanTree();
+        root.clearChildren();
+        tree.cleanup();
+        Iterator<Annotation> iter =tree.iterator();
+        assertFalse(iter.hasNext());
+        assertEquals(0, tree.numAnnotations());
+    }
+
+    @Test
+    public void assertRemoveChildrenIndexCleanupTest() {
+        populateSpanTree();
+        int no = tree.numAnnotations();
+        root.removeChildren(1);
+        tree.cleanup();
+        int postNo = tree.numAnnotations();
+        assertEquals((no-8), postNo);
+    }
+
+    @Test
+    public void assertClearChildrenIndexCleanupTest() {
+        populateSpanTree();
+        int no = tree.numAnnotations();
+        root.clearChildren(1);
+        tree.cleanup();
+        int postNo = tree.numAnnotations();
+        assertEquals(3, postNo);
+    }
+
+
+    @Test
+    public void assertASPLRemoveCleanupTest() {
+        populateSpanTree();
+        int no = tree.numAnnotations();
+        alternate2.removeChildren(2);
+        alternate2.removeChildren(1);
+        alternate2.remove(span33);
+        tree.cleanup();
+        int postNo = tree.numAnnotations();
+        assertEquals((no-6), postNo);
+    }
+
+
+    @Test
+    public void assertSPLRemoveCleanupTest() {
+        populateSpanTree();
+        int no = tree.numAnnotations();
+        branch.remove(alternate2);
+        tree.cleanup();
+        int postNo = tree.numAnnotations();
+        assertEquals((no-6), postNo);
+    }
+
+    @Test
+    public void assertIteratorRecursiveASPLTest() {
+        populateSpanTree();
+        int no = 0;
+        Iterator<Annotation> it = tree.iteratorRecursive(alternate2);
+        while (it.hasNext()) {
+            it.next();
+            no ++;
+        }
+        assertEquals(6, no);
+    }
+
+    @Test
+    public void assertClearAnnotationsRecursiveASPLTest() {
+        populateSpanTree();
+        int no = tree.numAnnotations();
+        tree.clearAnnotationsRecursive(alternate2);
+        int postNo = tree.numAnnotations();
+        assertEquals((no-6), postNo);
+    }
+
+
+
+    @Test (expected = IllegalStateException.class)
+    public void assertReuseRemovedNode() {
+        populateSpanTree();
+        root.remove(span3);
+        tree.annotate(span3, new Annotation(at1));
+    }
+
+    @Test (expected = IllegalStateException.class)
+    public void assertNeedForCleanup() {
+        populateSpanTree();
+        root.remove(span3);
+        Iterator<Annotation> iter =tree.iterator();
+        while (iter.hasNext()) {
+            SpanNode sn = iter.next().getSpanNode();
+        }
+    }
+
+    @Test
+    public void validateSpanTree() {
+        populateSpanTree();
+        int no = tree.numAnnotations();
+        root.remove(span3);
+        tree.cleanup();
+        int noAfter = tree.numAnnotations();
+        assertEquals((no-1), noAfter);
+    }
+
+    @After
+    public void tearDown() {
+        tree = null;
+    }
+
+
+    public void consumeAnnotations(SpanList root) {
+        if (root instanceof AlternateSpanList) {
+            parseAlternateLists((AlternateSpanList)root);
+            if (debug) System.out.println("\nGetting annotations for the SpanList itself : [" + root.getFrom() + ", " + root.getTo() + "] ");
+            getAnnotationsForNode(root);
+            return;
+        }
+        if (debug) System.out.println("\n\nSpanList: [" + root.getFrom() + ", " + root.getTo() + "] num Children: " + root.numChildren());
+        if (debug) System.out.println("-------------------");
+        Iterator<SpanNode> childIterator = root.childIterator();
+		while (childIterator.hasNext()) {
+			SpanNode node = childIterator.next();
+            //System.out.println("Span Node: " + node); // + " Span Text: " + node.getText(fieldValStr));
+            if (debug) System.out.println("\n\nSpan Node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+            if (node instanceof AlternateSpanList) {
+                parseAlternateLists((AlternateSpanList)node);
+                if (debug) System.out.println("---- Alternate SpanList complete ---");
+            } else if (node instanceof SpanList) {
+                if (debug) System.out.println("Encountered another span list");
+                SpanList spl = (SpanList) node;
+                ListIterator<SpanNode> lli = spl.childIterator();
+                while (lli.hasNext()) System.out.print(" " + lli.next() + " ");
+                consumeAnnotations((SpanList) node);
+            } else {
+       			if (debug) System.out.println("\nGetting annotations for this span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+   	    	    getAnnotationsForNode(node);
+            }
+		}
+        if (debug) System.out.println("\nGetting annotations for the SpanList itself : [" + root.getFrom() + ", " + root.getTo() + "] ");
+        getAnnotationsForNode(root);
+    }
+
+    public void parseAlternateLists(AlternateSpanList aspl) {
+        int no = aspl.getNumSubTrees();
+        if (debug) System.out.println("Parsing Alternate span list. No of subtrees: " + no);
+        int ctr = 0;
+        while (ctr < no) {
+            if (debug) System.out.println("\nSubTree: " + ctr + " probability: " + aspl.getProbability(ctr));
+            ListIterator<SpanNode> lIter = aspl.childIterator(ctr);
+            while (lIter.hasNext()) {
+                SpanNode spnNode = lIter.next();
+                if (debug) System.out.println("Parsing span node: [" + spnNode.getFrom() + ", " + spnNode.getTo() + "] ");
+                if (spnNode instanceof AlternateSpanList) {
+                    if (debug) System.out.println("A child alternate span list found. Recursing");
+                    parseAlternateLists((AlternateSpanList)spnNode);
+                } else if (spnNode instanceof SpanList) {
+                    if (debug) System.out.println("A child span list found. Recursing");
+                    consumeAnnotations((SpanList)spnNode);
+                } else {
+                    //System.out.println("Span Node (from alternate spanlist): " + spnNode);
+                    getAnnotationsForNode(spnNode);
+                }
+            }
+            ctr ++;
+        }
+    }
+
+
+    public void parseFieldForAnnotations(StringFieldValue sfv) {
+        Collection<SpanTree> c = sfv.getSpanTrees();
+        Iterator<SpanTree> iiter = c.iterator();
+        while (iiter.hasNext()) {
+            if (debug) System.out.println(sfv + " has annotations");
+            tree = iiter.next();
+            SpanList root = (SpanList) tree.getRoot();
+            consumeAnnotations(root);
+        }
+    }
+
+
+    public void getAnnotationsForNode(SpanNode node) {
+        Iterator<Annotation> iter = tree.iterator(node);
+        boolean annotationPresent = false;
+		while (iter.hasNext()) {
+            annotationPresent = true;
+			Annotation xx = iter.next();
+			AnnotationType t = xx.getType();
+			StringFieldValue fValue = (StringFieldValue) xx.getFieldValue();
+			if (debug) System.out.println("Annotation: " + xx);
+            if (fValue == null) {
+                if (debug) System.out.println("Field Value is null");
+                return;
+            } else {
+                if (debug) System.out.println("Field Value: " + fValue.getString());
+            }
+		}
+        if (!annotationPresent) {
+            if (debug) System.out.println("****No annotations found for the span node: [" + node.getFrom() + ", " + node.getTo() + "] ");
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanTreeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanTreeTestCase.java
new file mode 100755
index 00000000000..e61d8d10d8a
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SpanTreeTestCase.java
@@ -0,0 +1,880 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.*;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SpanTreeTestCase extends AbstractTypesTest {
+    SpanTree tree;
+    SpanList root;
+    SpanNode a;
+    SpanNode b;
+    SpanList c;
+    SpanNode d;
+    SpanNode e;
+    Annotation dummyA;
+    Annotation numberB;
+    Annotation bananaC;
+    Annotation appleC;
+    Annotation grapeC;
+    Annotation dummyD;
+    Annotation dummyE;
+
+    /**
+     * Sets up a simple tree:
+     * root:
+     * a (dummyA), b (numberB), c (bananaC, appleC, grapeC)
+     * c:
+     * d (dummyD), e (dummyE)
+     *
+     */
+    @Before
+    public void setUp() {
+        tree = new SpanTree("ballooo");
+        root = (SpanList) tree.getRoot();
+
+        a = new Span(1,1);
+        b = new Span(2,1);
+        c = new SpanList();
+        d = new Span(3,1);
+        e = new Span(4,1);
+
+        c.add(d).add(e);
+        root.add(a).add(b).add(c);
+
+        dummyA = new Annotation(dummy);
+        numberB = new Annotation(number);
+        bananaC = new Annotation(banana);
+        appleC = new Annotation(apple);
+        grapeC = new Annotation(grape);
+        dummyD = new Annotation(dummy);
+        dummyE = new Annotation(dummy);
+
+        tree.annotate(a, dummyA)
+                .annotate(b, numberB)
+                .annotate(c, bananaC)
+                .annotate(c, appleC)
+                .annotate(c, grapeC)
+                .annotate(d, dummyD)
+                .annotate(e, dummyE);
+    }
+
+    /**
+     * Tests iterating through all annotations in a tree.
+     */
+    @Test
+    public void testAnnotationIteration() {
+        List<Annotation> allAnnotationsList = new ArrayList<>();
+        for (Annotation a : tree) {
+            allAnnotationsList.add(a);
+        }
+        Collections.sort(allAnnotationsList);
+
+        Iterator<Annotation> allAnnotations = allAnnotationsList.iterator();
+
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+
+        assertSame(dummyA, allAnnotations.next());
+        assertSame(a, dummyA.getSpanNode());
+
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+
+        assertSame(numberB, allAnnotations.next());
+        assertSame(b, numberB.getSpanNode());
+
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+
+        assertSame(dummyD, allAnnotations.next());
+        assertSame(d, dummyD.getSpanNode());
+
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+
+        assertSame(grapeC, allAnnotations.next());
+        assertSame(c, grapeC.getSpanNode());
+
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+
+        assertSame(bananaC, allAnnotations.next());
+        assertSame(c, bananaC.getSpanNode());
+
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+
+        assertSame(appleC, allAnnotations.next());
+        assertSame(c, appleC.getSpanNode());
+
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+        assertTrue(allAnnotations.hasNext());
+
+
+        assertSame(dummyE, allAnnotations.next());
+        assertSame(e, dummyE.getSpanNode());
+
+        assertFalse(allAnnotations.hasNext());
+        assertFalse(allAnnotations.hasNext());
+        assertFalse(allAnnotations.hasNext());
+        assertFalse(allAnnotations.hasNext());
+    }
+
+    /**
+     * Tests iterating through A's annotations in a tree.
+     */
+    @Test
+    public void testAnnotationIterationForA() {
+        //A has no child nodes, should have equal results for recursive and non-recursive iterator
+        aIteration(tree.iterator(a));
+        aIteration(tree.iteratorRecursive(a));
+    }
+
+    private void aIteration(Iterator<Annotation> aAnnotations) {
+        assertTrue(aAnnotations.hasNext());
+        assertTrue(aAnnotations.hasNext());
+        assertTrue(aAnnotations.hasNext());
+        assertTrue(aAnnotations.hasNext());
+
+        assertSame(dummyA, aAnnotations.next());
+        assertSame(a, dummyA.getSpanNode());
+
+        assertFalse(aAnnotations.hasNext());
+        assertFalse(aAnnotations.hasNext());
+        assertFalse(aAnnotations.hasNext());
+        assertFalse(aAnnotations.hasNext());
+    }
+
+    /**
+     * Tests iterating through B's annotations in a tree.
+     */
+    @Test
+    public void testAnnotationIterationForB() {
+        //B has no child nodes, should have equal results for recursive and non-recursive iterator
+        bIteration(tree.iterator(b));
+        bIteration(tree.iteratorRecursive(b));
+    }
+
+    private void bIteration(Iterator<Annotation> bAnnotations) {
+        assertTrue(bAnnotations.hasNext());
+        assertTrue(bAnnotations.hasNext());
+        assertTrue(bAnnotations.hasNext());
+        assertTrue(bAnnotations.hasNext());
+
+        assertSame(numberB, bAnnotations.next());
+        assertSame(b, numberB.getSpanNode());
+
+        assertFalse(bAnnotations.hasNext());
+        assertFalse(bAnnotations.hasNext());
+        assertFalse(bAnnotations.hasNext());
+        assertFalse(bAnnotations.hasNext());
+    }
+
+    /**
+     * Tests iterating through C's annotations in a tree.
+     */
+    @Test
+    public void testAnnotationIterationForC() {
+        List<Annotation> cAnnotationsList = new ArrayList<>();
+        for (Iterator<Annotation> iteratorc = tree.iterator(c); iteratorc.hasNext(); ) {
+            cAnnotationsList.add(iteratorc.next());
+        }
+        Collections.sort(cAnnotationsList);
+
+        Iterator<Annotation> cAnnotations = cAnnotationsList.iterator();
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(grapeC, cAnnotations.next());
+        assertSame(c, grapeC.getSpanNode());
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(bananaC, cAnnotations.next());
+        assertSame(c, bananaC.getSpanNode());
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(appleC, cAnnotations.next());
+        assertSame(c, appleC.getSpanNode());
+
+        assertFalse(cAnnotations.hasNext());
+        assertFalse(cAnnotations.hasNext());
+        assertFalse(cAnnotations.hasNext());
+        assertFalse(cAnnotations.hasNext());
+    }
+
+    @Test
+    public void testRecursiveAnnotationIterationForC() {
+        List<Annotation> cAnnotationsList = new ArrayList<>();
+        for (Iterator<Annotation> iteratorc = tree.iteratorRecursive(c); iteratorc.hasNext(); ) {
+            cAnnotationsList.add(iteratorc.next());
+        }
+        Collections.sort(cAnnotationsList);
+
+        Iterator<Annotation> cAnnotations = cAnnotationsList.iterator();
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(dummyD, cAnnotations.next());
+        assertSame(d, dummyD.getSpanNode());
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(grapeC, cAnnotations.next());
+        assertSame(c, grapeC.getSpanNode());
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(bananaC, cAnnotations.next());
+        assertSame(c, bananaC.getSpanNode());
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(appleC, cAnnotations.next());
+        assertSame(c, appleC.getSpanNode());
+
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+        assertTrue(cAnnotations.hasNext());
+
+        assertSame(dummyE, cAnnotations.next());
+        assertSame(e, dummyE.getSpanNode());
+
+        assertFalse(cAnnotations.hasNext());
+        assertFalse(cAnnotations.hasNext());
+        assertFalse(cAnnotations.hasNext());
+        assertFalse(cAnnotations.hasNext());
+    }
+
+
+    /**
+     * Tests iterating through D's annotations in a tree.
+     */
+    @Test
+    public void testAnnotationIterationForD() {
+        //D has no child nodes, should have equal results for recursive and non-recursive iterator
+        dIteration(tree.iterator(d));
+        dIteration(tree.iteratorRecursive(d));
+    }
+
+    private void dIteration(Iterator<Annotation> dAnnotations) {
+        assertTrue(dAnnotations.hasNext());
+        assertTrue(dAnnotations.hasNext());
+        assertTrue(dAnnotations.hasNext());
+        assertTrue(dAnnotations.hasNext());
+
+        assertSame(dummyD, dAnnotations.next());
+        assertSame(d, dummyD.getSpanNode());
+
+        assertFalse(dAnnotations.hasNext());
+        assertFalse(dAnnotations.hasNext());
+        assertFalse(dAnnotations.hasNext());
+        assertFalse(dAnnotations.hasNext());
+    }
+
+
+    /**
+     * Tests iterating through E's annotations in a tree.
+     */
+    @Test
+    public void testAnnotationIterationForE() {
+        //E has no child nodes, should have equal results for recursive and non-recursive iterator
+        eIteration(tree.iterator(e));
+        eIteration(tree.iteratorRecursive(e));
+    }
+
+    private void eIteration(Iterator<Annotation> eAnnotations) {
+        assertTrue(eAnnotations.hasNext());
+        assertTrue(eAnnotations.hasNext());
+        assertTrue(eAnnotations.hasNext());
+        assertTrue(eAnnotations.hasNext());
+
+        assertSame(dummyE, eAnnotations.next());
+        assertSame(e, dummyE.getSpanNode());
+
+        assertFalse(eAnnotations.hasNext());
+        assertFalse(eAnnotations.hasNext());
+        assertFalse(eAnnotations.hasNext());
+        assertFalse(eAnnotations.hasNext());
+    }
+
+    @Test
+    public void testCleanup() {
+        SpanList a = new SpanList();
+        SpanTree tree = new SpanTree("ballooO", a);
+        Span b = new Span(0,1);
+        Span c = new Span(1,1);
+        a.add(b).add(c);
+
+        AnnotationReferenceDataType refTypeToDummy = new AnnotationReferenceDataType(dummy);
+        AnnotationType annotationTypeWithRefToDummy = new AnnotationType("reftodummy", refTypeToDummy);
+
+        AnnotationReferenceDataType refTypeToRefTypeToDummy = new AnnotationReferenceDataType(annotationTypeWithRefToDummy);
+        AnnotationType annotationTypeWithRefToRefToDummy = new AnnotationType("reftoreftodummy", refTypeToRefTypeToDummy);
+
+        StructDataType structType = new StructDataType("str");
+        Field refToDummyField = new Field("refToDummy", refTypeToDummy);
+        structType.addField(refToDummyField);
+        AnnotationType annotationTypeWithStruct = new AnnotationType("structwithref", structType);
+
+
+        Annotation dummyAnnotationForB = new Annotation(dummy);
+        tree.annotate(b, dummyAnnotationForB);
+
+        AnnotationReference referenceToDummyB = new AnnotationReference(refTypeToDummy, dummyAnnotationForB);
+        Annotation annotationWithRefToDummyB = new Annotation(annotationTypeWithRefToDummy, referenceToDummyB);
+        tree.annotate(annotationWithRefToDummyB);
+
+        AnnotationReference referenceToRefToDummyB = new AnnotationReference(refTypeToRefTypeToDummy, annotationWithRefToDummyB);
+        Annotation annotationWithRefToRefToDummyB = new Annotation(annotationTypeWithRefToRefToDummy, referenceToRefToDummyB);
+        tree.annotate(annotationWithRefToRefToDummyB);
+
+        Struct struct = new Struct(structType);
+        struct.setFieldValue(refToDummyField, new AnnotationReference(refTypeToDummy, dummyAnnotationForB));
+        Annotation annotationWithStruct = new Annotation(annotationTypeWithStruct, struct);
+        tree.annotate(annotationWithStruct);
+
+
+        Iterator<Annotation> annotationIt;
+
+        assertEquals(4, tree.numAnnotations());
+        assertTrue(a.isValid());
+        assertTrue(b.isValid());
+        assertTrue(c.isValid());
+        assertEquals(2, a.numChildren());
+
+        {
+            List<Annotation> allAnnotationsList = new ArrayList<>();
+            for (Annotation an : tree) {
+                allAnnotationsList.add(an);
+            }
+            Collections.sort(allAnnotationsList);
+
+            annotationIt = allAnnotationsList.iterator();
+        }
+        // NOTE: ordering of annotations is derived from their name
+
+        assertSame(annotationWithRefToDummyB, annotationIt.next());
+        assertFalse(annotationWithRefToDummyB.hasSpanNode());
+        assertFalse(annotationWithRefToDummyB.isSpanNodeValid());
+        assertTrue(annotationWithRefToDummyB.hasFieldValue());
+        assertSame(dummyAnnotationForB, ((AnnotationReference) annotationWithRefToDummyB.getFieldValue()).getReference());
+
+        assertSame(annotationWithStruct, annotationIt.next());
+        assertFalse(annotationWithStruct.hasSpanNode());
+        assertFalse(annotationWithStruct.isSpanNodeValid());
+        assertTrue(annotationWithStruct.hasFieldValue());
+        assertSame(struct, annotationWithStruct.getFieldValue());
+        assertSame(dummyAnnotationForB, ((AnnotationReference) struct.getFieldValue(refToDummyField)).getReference());
+
+        assertSame(annotationWithRefToRefToDummyB, annotationIt.next());
+        assertFalse(annotationWithRefToRefToDummyB.hasSpanNode());
+        assertFalse(annotationWithRefToRefToDummyB.isSpanNodeValid());
+        assertTrue(annotationWithRefToRefToDummyB.hasFieldValue());
+        assertSame(annotationWithRefToDummyB, ((AnnotationReference) annotationWithRefToRefToDummyB.getFieldValue()).getReference());
+
+        assertSame(dummyAnnotationForB, annotationIt.next());
+        assertTrue(dummyAnnotationForB.hasSpanNode());
+        assertTrue(dummyAnnotationForB.isSpanNodeValid());
+        assertFalse(dummyAnnotationForB.hasFieldValue());
+
+        assertFalse(annotationIt.hasNext());
+
+        //removing b!!
+        a.remove(b);
+
+
+        assertEquals(4, tree.numAnnotations());
+        assertTrue(a.isValid());
+        assertFalse(b.isValid());
+        assertTrue(c.isValid());
+        assertEquals(1, a.numChildren());
+
+        {
+            List<Annotation> allAnnotationsList = new ArrayList<>();
+            for (Annotation an : tree) {
+                allAnnotationsList.add(an);
+            }
+            Collections.sort(allAnnotationsList);
+
+            annotationIt = allAnnotationsList.iterator();
+        }
+
+        assertSame(annotationWithRefToDummyB, annotationIt.next());
+        assertFalse(annotationWithRefToDummyB.hasSpanNode());
+        assertFalse(annotationWithRefToDummyB.isSpanNodeValid());
+        assertTrue(annotationWithRefToDummyB.hasFieldValue());
+        assertSame(dummyAnnotationForB, ((AnnotationReference) annotationWithRefToDummyB.getFieldValue()).getReference());
+
+        assertSame(annotationWithStruct, annotationIt.next());
+        assertFalse(annotationWithStruct.hasSpanNode());
+        assertFalse(annotationWithStruct.isSpanNodeValid());
+        assertTrue(annotationWithStruct.hasFieldValue());
+        assertSame(struct, annotationWithStruct.getFieldValue());
+        assertSame(dummyAnnotationForB, ((AnnotationReference) struct.getFieldValue(refToDummyField)).getReference());
+
+        assertSame(annotationWithRefToRefToDummyB, annotationIt.next());
+        assertFalse(annotationWithRefToRefToDummyB.hasSpanNode());
+        assertFalse(annotationWithRefToRefToDummyB.isSpanNodeValid());
+        assertTrue(annotationWithRefToRefToDummyB.hasFieldValue());
+        assertSame(annotationWithRefToDummyB, ((AnnotationReference) annotationWithRefToRefToDummyB.getFieldValue()).getReference());
+
+        assertSame(dummyAnnotationForB, annotationIt.next());
+        assertTrue(dummyAnnotationForB.hasSpanNode());
+        assertFalse(dummyAnnotationForB.isSpanNodeValid());
+        assertFalse(dummyAnnotationForB.hasFieldValue());
+
+        assertFalse(annotationIt.hasNext());
+
+
+        //cleaning up tree!!
+        tree.cleanup();
+
+        assertEquals(1, tree.numAnnotations());
+        assertTrue(a.isValid());
+        assertFalse(b.isValid());
+        assertTrue(c.isValid());
+        assertEquals(1, a.numChildren());
+
+        assertFalse(dummyAnnotationForB.hasSpanNode());
+        assertFalse(dummyAnnotationForB.isSpanNodeValid());
+        assertFalse(dummyAnnotationForB.hasFieldValue());
+
+        assertFalse(annotationWithRefToDummyB.hasSpanNode());
+        assertFalse(annotationWithRefToDummyB.isSpanNodeValid());
+        assertFalse(annotationWithRefToDummyB.hasFieldValue());
+
+        assertFalse(annotationWithRefToRefToDummyB.hasSpanNode());
+        assertFalse(annotationWithRefToRefToDummyB.isSpanNodeValid());
+        assertFalse(annotationWithRefToRefToDummyB.hasFieldValue());
+
+        annotationIt = tree.iterator();
+        assertSame(annotationWithStruct, annotationIt.next());
+        assertFalse(annotationWithStruct.hasSpanNode());
+        assertFalse(annotationWithStruct.isSpanNodeValid());
+        assertTrue(annotationWithStruct.hasFieldValue());
+        assertSame(struct, annotationWithStruct.getFieldValue());
+        assertNull(struct.getFieldValue(refToDummyField));
+
+        assertFalse(annotationIt.hasNext());
+    }
+
+    @Test
+    public void testSimpleCopy() {
+        StringFieldValue string = new StringFieldValue("yahoooo");
+
+        SpanList list = new SpanList();
+        Span span1 = new Span(0,1);
+        Span span2 = new Span(1,1);
+        list.add(span1).add(span2);
+        SpanTree tree = new SpanTree("foo", list);
+        string.setSpanTree(tree);
+
+        Annotation a1 = new Annotation(grape);
+        tree.annotate(span1, a1);
+        Annotation a2 = new Annotation(apple);
+        tree.annotate(span2, a2);
+
+        StringFieldValue stringCopy = string.clone();
+
+
+
+        assertEquals(string, stringCopy);
+        assertNotSame(string, stringCopy);
+
+        SpanTree treeCopy = stringCopy.getSpanTree("foo");
+        assertEquals(tree, treeCopy);
+        assertNotSame(tree, treeCopy);
+
+        SpanList listCopy = (SpanList) treeCopy.getRoot();
+        assertEquals(list, listCopy);
+        assertNotSame(list, listCopy);
+
+        Span span1Copy = (Span) listCopy.children().get(0);
+        assertEquals(span1, span1Copy);
+        assertNotSame(span1, span1Copy);
+
+        Span span2Copy = (Span) listCopy.children().get(1);
+        assertEquals(span2, span2Copy);
+        assertNotSame(span2, span2Copy);
+
+        Iterator<Annotation> annotationIt;
+        {
+            List<Annotation> allAnnotationsList = new ArrayList<>();
+            for (Annotation an : treeCopy) {
+                allAnnotationsList.add(an);
+            }
+            Collections.sort(allAnnotationsList);
+
+            annotationIt = allAnnotationsList.iterator();
+        }
+
+        Annotation a1Copy = annotationIt.next();
+        assertEquals(a1, a1Copy);
+        assertNotSame(a1, a1Copy);
+        assertSame(span1Copy, a1Copy.getSpanNode());
+        assertNotSame(span1, a1Copy.getSpanNode());
+
+        Annotation a2Copy = annotationIt.next();
+        assertEquals(a2, a2Copy);
+        assertNotSame(a2, a2Copy);
+        assertSame(span2Copy, a2Copy.getSpanNode());
+        assertNotSame(span2, a2Copy.getSpanNode());
+    }
+
+    @Test
+    public void testSimpleCopy2() {
+        StringFieldValue string = new StringFieldValue("yahoooo");
+
+        SpanList list = new SpanList();
+        Span span1 = new Span(0,1);
+        Span span2 = new Span(1,1);
+        list.add(span1).add(span2);
+        SpanTree tree = new SpanTree("foo", list);
+        string.setSpanTree(tree);
+
+        Annotation a1 = new Annotation(grape);
+        tree.annotate(span1, a1);
+        Annotation a2 = new Annotation(apple);
+        tree.annotate(span2, a2);
+
+        Struct donald = new Struct(person);
+        donald.setFieldValue("firstname", "donald");
+        donald.setFieldValue("lastname", "duck");
+        donald.setFieldValue("birthyear", 1929);
+        Annotation donaldAnn = new Annotation(personA, donald);
+        tree.annotate(list, donaldAnn);
+
+
+        StringFieldValue stringCopy = string.clone();
+
+
+
+        assertEquals(string, stringCopy);
+        assertNotSame(string, stringCopy);
+
+        SpanTree treeCopy = stringCopy.getSpanTree("foo");
+        assertEquals(tree, treeCopy);
+        assertNotSame(tree, treeCopy);
+
+        SpanList listCopy = (SpanList) treeCopy.getRoot();
+        assertEquals(list, listCopy);
+        assertNotSame(list, listCopy);
+
+        Span span1Copy = (Span) listCopy.children().get(0);
+        assertEquals(span1, span1Copy);
+        assertNotSame(span1, span1Copy);
+
+        Span span2Copy = (Span) listCopy.children().get(1);
+        assertEquals(span2, span2Copy);
+        assertNotSame(span2, span2Copy);
+
+        Iterator<Annotation> annotationIt;
+        {
+            List<Annotation> allAnnotationsList = new ArrayList<>();
+            for (Annotation an : treeCopy) {
+                allAnnotationsList.add(an);
+            }
+            Collections.sort(allAnnotationsList);
+
+            annotationIt = allAnnotationsList.iterator();
+        }
+
+        Annotation a1Copy = annotationIt.next();
+        assertEquals(a1, a1Copy);
+        assertNotSame(a1, a1Copy);
+        assertSame(span1Copy, a1Copy.getSpanNode());
+        assertNotSame(span1, a1Copy.getSpanNode());
+
+        Annotation donaldAnnCopy = annotationIt.next();
+        assertEquals(donaldAnn, donaldAnnCopy);
+        assertNotSame(donaldAnn, donaldAnnCopy);
+        assertSame(listCopy, donaldAnnCopy.getSpanNode());
+        assertNotSame(list, donaldAnnCopy.getSpanNode());
+
+        Struct donaldCopy = (Struct) donaldAnnCopy.getFieldValue();
+        assertEquals(donald, donaldCopy);
+        assertNotSame(donald, donaldCopy);
+
+        Annotation a2Copy = annotationIt.next();
+        assertEquals(a2, a2Copy);
+        assertNotSame(a2, a2Copy);
+        assertSame(span2Copy, a2Copy.getSpanNode());
+        assertNotSame(span2, a2Copy.getSpanNode());
+    }
+
+    @Test
+    public void testCopyAnnotatedString() {
+        StringFieldValue str = getAnnotatedString();
+        StringFieldValue strCopy = str.clone();
+
+        SpanTree balloooTree = str.getSpanTree("ballooo");
+        AlternateSpanList root = (AlternateSpanList) balloooTree.getRoot();
+        Span s1 = (Span) root.children(0).get(0);
+        Span s2 = (Span) root.children(0).get(1);
+        Span s3 = (Span) root.children(0).get(2);
+        AlternateSpanList s4 = (AlternateSpanList) root.children(0).get(3);
+        Span s5 = (Span) s4.children(0).get(0);
+        Span s6 = (Span) s4.children(0).get(1);
+        Span s7 = (Span) root.children(1).get(0);
+        Span s8 = (Span) root.children(1).get(1);
+        Span s9 = (Span) root.children(1).get(2);
+
+        SpanTree balloooTreeCopy = strCopy.getSpanTree("ballooo");
+        assertEquals(balloooTree, balloooTreeCopy);
+        assertNotSame(balloooTree, balloooTreeCopy);
+        AlternateSpanList rootCopy = (AlternateSpanList) balloooTreeCopy.getRoot();
+        assertEquals(root, rootCopy);
+        assertNotSame(root, rootCopy);
+        Span s1Copy = (Span) rootCopy.children(0).get(0);
+        assertEquals(s1, s1Copy);
+        assertNotSame(s1, s1Copy);
+        Span s2Copy = (Span) rootCopy.children(0).get(1);
+        assertEquals(s2, s2Copy);
+        assertNotSame(s2, s2Copy);
+        Span s3Copy = (Span) rootCopy.children(0).get(2);
+        assertEquals(s3, s3Copy);
+        assertNotSame(s3, s3Copy);
+        AlternateSpanList s4Copy = (AlternateSpanList) rootCopy.children(0).get(3);
+        assertEquals(s4, s4Copy);
+        assertNotSame(s4, s4Copy);
+        Span s5Copy = (Span) s4Copy.children(0).get(0);
+        assertEquals(s5, s5Copy);
+        assertNotSame(s5, s5Copy);
+        Span s6Copy = (Span) s4Copy.children(0).get(1);
+        assertEquals(s6, s6Copy);
+        assertNotSame(s6, s6Copy);
+        Span s7Copy = (Span) rootCopy.children(1).get(0);
+        assertEquals(s7, s7Copy);
+        assertNotSame(s7, s7Copy);
+        Span s8Copy = (Span) rootCopy.children(1).get(1);
+        assertEquals(s8, s8Copy);
+        assertNotSame(s8, s8Copy);
+        Span s9Copy = (Span) rootCopy.children(1).get(2);
+        assertEquals(s9, s9Copy);
+        assertNotSame(s9, s9Copy);
+
+
+        Iterator<Annotation> annotationsBalloooTree;
+        {
+            List<Annotation> allAnnotationsList = new ArrayList<>();
+            for (Annotation an : balloooTree) {
+                allAnnotationsList.add(an);
+            }
+            Collections.sort(allAnnotationsList);
+
+            annotationsBalloooTree = allAnnotationsList.iterator();
+        }
+
+        Annotation dummyAnnForS1 = annotationsBalloooTree.next();
+
+        Annotation dummyAnnForS2 = annotationsBalloooTree.next();
+
+        Annotation numberAnnForS2 = annotationsBalloooTree.next();
+        IntegerFieldValue integerValForS2 = (IntegerFieldValue) numberAnnForS2.getFieldValue();
+
+        Annotation motherAnnForS2 = annotationsBalloooTree.next();
+        Struct motherValForS2 = (Struct) motherAnnForS2.getFieldValue();
+
+        Annotation dummyAnnForS3 = annotationsBalloooTree.next();
+
+        Annotation numberAnnForS3 = annotationsBalloooTree.next();
+        IntegerFieldValue integerValForS3 = (IntegerFieldValue) numberAnnForS3.getFieldValue();
+
+        Annotation dummyAnnForS5 = annotationsBalloooTree.next();
+
+        Annotation daughterAnnForS6 = annotationsBalloooTree.next();
+        Struct daughterValForS6 = (Struct) daughterAnnForS6.getFieldValue();
+        AnnotationReference refFromS6ToMotherAnn = (AnnotationReference) daughterValForS6.getFieldValue("related");
+
+
+        Iterator<Annotation> annotationsBalloooTreeCopy;
+        {
+            List<Annotation> allAnnotationsList = new ArrayList<>();
+            for (Annotation an : balloooTreeCopy) {
+                allAnnotationsList.add(an);
+            }
+            Collections.sort(allAnnotationsList);
+
+            annotationsBalloooTreeCopy = allAnnotationsList.iterator();
+        }
+
+        Annotation dummyAnnForS1Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(dummyAnnForS1, dummyAnnForS1Copy);
+        assertNotSame(dummyAnnForS1, dummyAnnForS1Copy);
+
+        Annotation dummyAnnForS2Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(dummyAnnForS2, dummyAnnForS2Copy);
+        assertNotSame(dummyAnnForS2, dummyAnnForS2Copy);
+
+        Annotation numberAnnForS2Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(numberAnnForS2, numberAnnForS2Copy);
+        assertNotSame(numberAnnForS2, numberAnnForS2Copy);
+        IntegerFieldValue integerValForS2Copy = (IntegerFieldValue) numberAnnForS2Copy.getFieldValue();
+        assertEquals(integerValForS2, integerValForS2Copy);
+        assertNotSame(integerValForS2, integerValForS2Copy);
+
+        Annotation motherAnnForS2Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(motherAnnForS2, motherAnnForS2Copy);
+        assertNotSame(motherAnnForS2, motherAnnForS2Copy);
+        Struct motherValForS2Copy = (Struct) motherAnnForS2Copy.getFieldValue();
+        assertEquals(motherValForS2, motherValForS2Copy);
+        assertNotSame(motherValForS2, motherValForS2Copy);
+
+        Annotation dummyAnnForS3Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(dummyAnnForS3, dummyAnnForS3Copy);
+        assertNotSame(dummyAnnForS3, dummyAnnForS3Copy);
+
+        Annotation numberAnnForS3Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(numberAnnForS3, numberAnnForS3Copy);
+        assertNotSame(numberAnnForS3, numberAnnForS3Copy);
+        IntegerFieldValue integerValForS3Copy = (IntegerFieldValue) numberAnnForS3Copy.getFieldValue();
+        assertEquals(integerValForS3, integerValForS3Copy);
+        assertNotSame(integerValForS3, integerValForS3Copy);
+
+        Annotation dummyAnnForS5Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(dummyAnnForS5, dummyAnnForS5Copy);
+        assertNotSame(dummyAnnForS5, dummyAnnForS5Copy);
+
+        Annotation daughterAnnForS6Copy = annotationsBalloooTreeCopy.next();
+        assertEquals(daughterAnnForS6, daughterAnnForS6Copy);
+        assertNotSame(daughterAnnForS6, daughterAnnForS6Copy);
+        Struct daughterValForS6Copy = (Struct) daughterAnnForS6Copy.getFieldValue();
+        assertEquals(daughterValForS6, daughterValForS6Copy);
+        assertNotSame(daughterValForS6, daughterValForS6Copy);
+        AnnotationReference refFromS6ToMotherAnnCopy = (AnnotationReference) daughterValForS6Copy.getFieldValue("related");
+        assertEquals(refFromS6ToMotherAnn, refFromS6ToMotherAnnCopy);
+        assertNotSame(refFromS6ToMotherAnn, refFromS6ToMotherAnnCopy);
+
+        assertEquals(str, strCopy);
+
+    }
+
+    @Test
+    public void testCyclicReferences() {
+        DocumentTypeManager docMan = new DocumentTypeManager();
+        AnnotationTypeRegistry reg = docMan.getAnnotationTypeRegistry();
+
+        StringFieldValue strfval = new StringFieldValue("hohohoho");
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("habahaba", root);
+        strfval.setSpanTree(tree);
+
+        //set up types:
+        AnnotationType endTagType = new AnnotationType("endtag");
+        AnnotationType beginTagType = new AnnotationType("begintag");
+        AnnotationReferenceDataType refToBeginTagDataType = new AnnotationReferenceDataType(beginTagType);
+        AnnotationReferenceDataType refToEndTagDataType = new AnnotationReferenceDataType(endTagType);
+        endTagType.setDataType(refToBeginTagDataType);
+        beginTagType.setDataType(refToEndTagDataType);
+
+        //register types:
+        reg.register(endTagType);
+        reg.register(beginTagType);
+        docMan.register(refToBeginTagDataType);
+        docMan.register(refToEndTagDataType);
+
+        //set up annotations and their references to each other:
+        AnnotationReference refToBeginTag = new AnnotationReference(refToBeginTagDataType);
+        AnnotationReference refToEndTag = new AnnotationReference(refToEndTagDataType);
+        Annotation beginTag = new Annotation(beginTagType, refToEndTag);
+        Annotation endTag = new Annotation(endTagType, refToBeginTag);
+        refToBeginTag.setReference(beginTag);
+        refToEndTag.setReference(endTag);
+
+        //create spans:
+        Span beginTagSpan = new Span(0, 1);
+        Span endTagSpan = new Span(1, 1);
+        root.add(beginTagSpan);
+        root.add(endTagSpan);
+
+        //annotate spans:
+        tree.annotate(beginTagSpan, beginTag);
+        tree.annotate(endTagSpan, endTag);
+
+
+        //none of the below statements should lead to a StackOverflowError:
+        assertFalse(endTag.toString().equals(beginTag.toString()));
+
+        assertTrue(endTag.hashCode() != beginTag.hashCode());
+
+        assertFalse(endTag.equals(beginTag));
+        assertFalse(beginTag.equals(endTag));
+
+        StringFieldValue copyString = strfval.clone();
+        assertEquals(strfval, copyString);
+
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+
+        serializer.write(new Field("stringfield", DataType.STRING), strfval);
+        buffer.flip();
+
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(docMan, buffer);
+        StringFieldValue stringFieldValue2 = new StringFieldValue();
+        deserializer.read(new Field("stringfield", DataType.STRING), stringFieldValue2);
+
+        assertEquals(strfval, stringFieldValue2);
+        assertNotSame(strfval, stringFieldValue2);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/SystemTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SystemTestCase.java
new file mode 100755
index 00000000000..e8afa472233
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/SystemTestCase.java
@@ -0,0 +1,130 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.annotation;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+
+import java.util.Iterator;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SystemTestCase extends junit.framework.TestCase {
+    DocumentTypeManager manager;
+
+    private void annotate(Document document) {
+        AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();
+        AnnotationType personType = registry.getType("person");
+        AnnotationType artistType = registry.getType("artist");
+        AnnotationType dateType = registry.getType("date");
+        AnnotationType placeType = registry.getType("place");
+        AnnotationType eventType = registry.getType("event");
+
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("meaningoflife", root);
+
+        SpanNode personSpan = new Span(0,5);
+        SpanNode artistSpan = new Span(5,10);
+        SpanNode dateSpan = new Span(10,15);
+        SpanNode placeSpan = new Span(15,20);
+
+        root.add(personSpan);
+        root.add(artistSpan);
+        root.add(dateSpan);
+        root.add(placeSpan);
+
+        Struct personValue = new Struct(manager.getDataType("annotation.person"));
+        personValue.setFieldValue("name", "george washington");
+        Annotation person = new Annotation(personType, personValue);
+        tree.annotate(personSpan, person);
+
+        Struct artistValue = new Struct(manager.getDataType("annotation.artist"));
+        artistValue.setFieldValue("name", "elvis presley");
+        artistValue.setFieldValue("instrument", 20);
+        Annotation artist = new Annotation(artistType, artistValue);
+        tree.annotate(artistSpan, artist);
+
+        Struct dateValue = new Struct(manager.getDataType("annotation.date"));
+        dateValue.setFieldValue("exacttime", 123456789L);
+        Annotation date = new Annotation(dateType, dateValue);
+        tree.annotate(dateSpan, date);
+
+        Struct placeValue = new Struct(manager.getDataType("annotation.place"));
+        placeValue.setFieldValue("lat", 1467L);
+        placeValue.setFieldValue("lon", 789L);
+        Annotation place = new Annotation(placeType, placeValue);
+        tree.annotate(placeSpan, place);
+
+        Struct eventValue = new Struct(manager.getDataType("annotation.event"));
+        eventValue.setFieldValue("description", "Big concert");
+        eventValue.setFieldValue("person", new AnnotationReference((AnnotationReferenceDataType) manager.getDataType("annotationreference<person>"), person));
+        eventValue.setFieldValue("date", new AnnotationReference((AnnotationReferenceDataType) manager.getDataType("annotationreference<date>"), date));
+        eventValue.setFieldValue("place", new AnnotationReference((AnnotationReferenceDataType) manager.getDataType("annotationreference<place>"), place));
+        Annotation event = new Annotation(eventType, eventValue);
+        tree.annotate(root, event);
+
+        StringFieldValue content = new StringFieldValue("This is the story of a big concert by Elvis and a special guest appearance by George Washington");
+        content.setSpanTree(tree);
+
+        document.setFieldValue(document.getDataType().getField("content"), content);
+    }
+
+    private void consume(Document document) {
+        StringFieldValue content = (StringFieldValue) document.getFieldValue(document.getDataType().getField("content"));
+
+        SpanTree tree = content.getSpanTree("meaningoflife");
+        SpanList root = (SpanList) tree.getRoot();
+
+        Iterator<SpanNode> childIterator = root.childIterator();
+        SpanNode personSpan = childIterator.next();
+        SpanNode artistSpan = childIterator.next();
+        SpanNode dateSpan = childIterator.next();
+        SpanNode placeSpan = childIterator.next();
+
+        Annotation person = tree.iterator(personSpan).next();
+        Struct personValue = (Struct) person.getFieldValue();
+        System.err.println("Person is " + personValue.getField("name"));
+
+        Annotation artist = tree.iterator(artistSpan).next();
+        Struct artistValue = (Struct) artist.getFieldValue();
+        System.err.println("Artist is " + artistValue.getFieldValue("name") + " who plays the " + artistValue.getFieldValue("instrument"));
+
+        Annotation date = tree.iterator(dateSpan).next();
+        Struct dateValue = (Struct) date.getFieldValue();
+        System.err.println("Date is " + dateValue.getFieldValue("exacttime"));
+
+        Annotation place = tree.iterator(placeSpan).next();
+        Struct placeValue = (Struct) place.getFieldValue();
+        System.err.println("Place is " + placeValue.getFieldValue("lat") + ";" + placeValue.getFieldValue("lon"));
+
+        Annotation event = tree.iterator(root).next();
+        Struct eventValue = (Struct) event.getFieldValue();
+        System.err.println("Event is " + eventValue.getFieldValue("description") + " with " + eventValue.getFieldValue("person") + " and " + eventValue.getFieldValue("date") + " and " + eventValue.getFieldValue("place"));
+    }
+
+    public void setUp() {
+        manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/java/com/yahoo/document/annotation/documentmanager.systemtest.cfg");
+    }
+
+    public void testSystemTest() {
+        DocumentType type = manager.getDocumentType("article");
+        Document inDocument = new Document(type, "doc:article:boringarticle:longarticle");
+        annotate(inDocument);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer();
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+        serializer.write(inDocument);
+        buffer.flip();
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer);
+
+        Document outDocument = new Document(deserializer);
+        consume(outDocument);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/annotation/documentmanager.6394548.cfg b/document/src/test/java/com/yahoo/document/annotation/documentmanager.6394548.cfg
new file mode 100644
index 00000000000..35717a3e0c1
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/documentmanager.6394548.cfg
@@ -0,0 +1,189 @@
+enablecompression false
+datatype[0].id 1381038251
+datatype[0].structtype[0].name "position"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].compresstype NONE
+datatype[0].structtype[0].compresslevel 0
+datatype[0].structtype[0].compressthreshold 95
+datatype[0].structtype[0].compressminsize 800
+datatype[0].structtype[0].field[0].name "x"
+datatype[0].structtype[0].field[0].datatype 0
+datatype[0].structtype[0].field[1].name "y"
+datatype[0].structtype[0].field[1].datatype 0
+datatype[1].id -1466283082
+datatype[1].structtype[0].name "annotation.person"
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].compresstype NONE
+datatype[1].structtype[0].compresslevel 0
+datatype[1].structtype[0].compressthreshold 95
+datatype[1].structtype[0].compressminsize 800
+datatype[1].structtype[0].field[0].name "name"
+datatype[1].structtype[0].field[0].datatype 2
+datatype[2].id -1149562679
+datatype[2].annotationreftype[0].annotation "person"
+datatype[3].id -772171888
+datatype[3].annotationreftype[0].annotation "date"
+datatype[4].id -2109350185
+datatype[4].annotationreftype[0].annotation "place"
+datatype[5].id 1194300957
+datatype[5].structtype[0].name "annotation.event"
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].compresstype NONE
+datatype[5].structtype[0].compresslevel 0
+datatype[5].structtype[0].compressthreshold 95
+datatype[5].structtype[0].compressminsize 800
+datatype[5].structtype[0].field[0].name "description"
+datatype[5].structtype[0].field[0].datatype 2
+datatype[5].structtype[0].field[1].name "person"
+datatype[5].structtype[0].field[1].datatype -1149562679
+datatype[5].structtype[0].field[2].name "date"
+datatype[5].structtype[0].field[2].datatype -772171888
+datatype[5].structtype[0].field[3].name "place"
+datatype[5].structtype[0].field[3].datatype -2109350185
+datatype[6].id 1463704666
+datatype[6].structtype[0].name "annotation.morty.RICK_DOCSTATS"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].compresstype NONE
+datatype[6].structtype[0].compresslevel 0
+datatype[6].structtype[0].compressthreshold 95
+datatype[6].structtype[0].compressminsize 800
+datatype[6].structtype[0].field[0].name "bodycount"
+datatype[6].structtype[0].field[0].datatype 0
+datatype[6].structtype[0].field[1].name "anchorcount"
+datatype[6].structtype[0].field[1].datatype 0
+datatype[7].id 1157126952
+datatype[7].structtype[0].name "annotation.artist"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].compresstype NONE
+datatype[7].structtype[0].compresslevel 0
+datatype[7].structtype[0].compressthreshold 95
+datatype[7].structtype[0].compressminsize 800
+datatype[7].structtype[0].field[0].name "instrument"
+datatype[7].structtype[0].field[0].datatype 0
+datatype[7].structtype[0].inherits[0].name "annotation.person"
+datatype[7].structtype[0].inherits[0].version 0
+datatype[8].id 2076579146
+datatype[8].structtype[0].name "annotation.place"
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].compresstype NONE
+datatype[8].structtype[0].compresslevel 0
+datatype[8].structtype[0].compressthreshold 95
+datatype[8].structtype[0].compressminsize 800
+datatype[8].structtype[0].field[0].name "lat"
+datatype[8].structtype[0].field[0].datatype 4
+datatype[8].structtype[0].field[1].name "lon"
+datatype[8].structtype[0].field[1].datatype 4
+datatype[9].id -840345201
+datatype[9].structtype[0].name "annotation.date"
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].compresstype NONE
+datatype[9].structtype[0].compresslevel 0
+datatype[9].structtype[0].compressthreshold 95
+datatype[9].structtype[0].compressminsize 800
+datatype[9].structtype[0].field[0].name "exacttime"
+datatype[9].structtype[0].field[0].datatype 4
+datatype[10].id -1486737430
+datatype[10].arraytype[0].datatype 2
+datatype[11].id 1021048351
+datatype[11].structtype[0].name "annotation.morty.RICK_FEATURESET"
+datatype[11].structtype[0].version 0
+datatype[11].structtype[0].compresstype NONE
+datatype[11].structtype[0].compresslevel 0
+datatype[11].structtype[0].compressthreshold 95
+datatype[11].structtype[0].compressminsize 800
+datatype[11].structtype[0].field[0].name "foo1"
+datatype[11].structtype[0].field[0].datatype 2
+datatype[11].structtype[0].field[1].name "foo2"
+datatype[11].structtype[0].field[1].datatype 0
+datatype[11].structtype[0].field[2].name "foo3"
+datatype[11].structtype[0].field[2].datatype 0
+datatype[11].structtype[0].field[3].name "foo4"
+datatype[11].structtype[0].field[3].datatype 2
+datatype[11].structtype[0].field[4].name "foo5"
+datatype[11].structtype[0].field[4].datatype 2
+datatype[11].structtype[0].field[5].name "foo6"
+datatype[11].structtype[0].field[5].datatype -1486737430
+datatype[11].structtype[0].field[6].name "foo7"
+datatype[11].structtype[0].field[6].datatype 0
+datatype[11].structtype[0].field[7].name "foo8"
+datatype[11].structtype[0].field[7].datatype 0
+datatype[11].structtype[0].field[8].name "foo9"
+datatype[11].structtype[0].field[8].datatype 1
+datatype[11].structtype[0].field[9].name "foo10"
+datatype[11].structtype[0].field[9].datatype -1486737430
+datatype[11].structtype[0].inherits[0].name "annotation.morty.FEATURESET"
+datatype[11].structtype[0].inherits[0].version 0
+datatype[12].id -228273582
+datatype[12].maptype[0].keytype 2
+datatype[12].maptype[0].valtype 5
+datatype[13].id -1584287606
+datatype[13].maptype[0].keytype 2
+datatype[13].maptype[0].valtype 0
+datatype[14].id 1980242844
+datatype[14].structtype[0].name "annotation.morty.FEATURESET"
+datatype[14].structtype[0].version 0
+datatype[14].structtype[0].compresstype NONE
+datatype[14].structtype[0].compresslevel 0
+datatype[14].structtype[0].compressthreshold 95
+datatype[14].structtype[0].compressminsize 800
+datatype[14].structtype[0].field[0].name "realvaluedfeatures"
+datatype[14].structtype[0].field[0].datatype -228273582
+datatype[14].structtype[0].field[1].name "discretevaluedfeatures"
+datatype[14].structtype[0].field[1].datatype -1584287606
+datatype[14].structtype[0].field[2].name "score"
+datatype[14].structtype[0].field[2].datatype 5
+datatype[15].id 892457735
+datatype[15].structtype[0].name "article.header"
+datatype[15].structtype[0].version 0
+datatype[15].structtype[0].compresstype NONE
+datatype[15].structtype[0].compresslevel 0
+datatype[15].structtype[0].compressthreshold 95
+datatype[15].structtype[0].compressminsize 800
+datatype[15].structtype[0].field[0].name "title"
+datatype[15].structtype[0].field[0].datatype 2
+datatype[15].structtype[0].field[1].name "content"
+datatype[15].structtype[0].field[1].datatype 2
+datatype[15].structtype[0].field[2].name "rankfeatures"
+datatype[15].structtype[0].field[2].datatype 2
+datatype[15].structtype[0].field[3].name "summaryfeatures"
+datatype[15].structtype[0].field[3].datatype 2
+datatype[16].id -1984964900
+datatype[16].structtype[0].name "article.body"
+datatype[16].structtype[0].version 0
+datatype[16].structtype[0].compresstype NONE
+datatype[16].structtype[0].compresslevel 0
+datatype[16].structtype[0].compressthreshold 95
+datatype[16].structtype[0].compressminsize 800
+datatype[17].id 559508792
+datatype[17].documenttype[0].name "article"
+datatype[17].documenttype[0].version 0
+datatype[17].documenttype[0].inherits[0].name "document"
+datatype[17].documenttype[0].inherits[0].version 0
+datatype[17].documenttype[0].headerstruct 892457735
+datatype[17].documenttype[0].bodystruct -1984964900
+annotationtype[0].id 609952424
+annotationtype[0].name "person"
+annotationtype[0].datatype -1466283082
+annotationtype[1].id -455530995
+annotationtype[1].name "event"
+annotationtype[1].datatype 1194300957
+annotationtype[2].id 295631537
+annotationtype[2].name "morty.RICK_DOCSTATS"
+annotationtype[2].datatype 1463704666
+annotationtype[3].id 690330276
+annotationtype[3].name "artist"
+annotationtype[3].datatype 1157126952
+annotationtype[3].inherits[0].id 609952424
+annotationtype[4].id -162455681
+annotationtype[4].name "date"
+annotationtype[4].datatype -840345201
+annotationtype[5].id 1707984040
+annotationtype[5].name "place"
+annotationtype[5].datatype 2076579146
+annotationtype[6].id -62680437
+annotationtype[6].name "morty.FEATURESET"
+annotationtype[6].datatype 1980242844
+annotationtype[7].id 1056322897
+annotationtype[7].name "morty.RICK_FEATURESET"
+annotationtype[7].datatype 1021048351
+annotationtype[7].inherits[0].id -62680437
diff --git a/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4259784.cfg b/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4259784.cfg
new file mode 100644
index 00000000000..77d72ed264e
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4259784.cfg
@@ -0,0 +1,147 @@
+enablecompression false
+annotationtype[4]
+annotationtype[0].datatype -476092672
+annotationtype[0].id 1278713514
+annotationtype[0].name "company"
+annotationtype[0].inherits[1]
+annotationtype[0].inherits[0].id 9765800
+annotationtype[1].datatype 912259135
+annotationtype[1].id 9765800
+annotationtype[1].name "industry"
+annotationtype[1].inherits[0]
+annotationtype[2].datatype 515587158
+annotationtype[2].id -270471211
+annotationtype[2].name "location"
+annotationtype[2].inherits[0]
+annotationtype[3].datatype -1466283082
+annotationtype[3].id 609952424
+annotationtype[3].name "person"
+annotationtype[3].inherits[0]
+datatype[10]
+datatype[0].id -1149562679
+datatype[0].annotationreftype[1]
+datatype[0].annotationreftype[0].annotation "person"
+datatype[0].arraytype[0]
+datatype[0].documenttype[0]
+datatype[0].structtype[0]
+datatype[0].weightedsettype[0]
+datatype[1].id -1386162972
+datatype[1].annotationreftype[0]
+datatype[1].arraytype[0]
+datatype[1].documenttype[1]
+datatype[1].documenttype[0].bodystruct 1387420336
+datatype[1].documenttype[0].headerstruct -945638949
+datatype[1].documenttype[0].name "blog"
+datatype[1].documenttype[0].version 0
+datatype[1].documenttype[0].inherits[0]
+datatype[1].structtype[0]
+datatype[1].weightedsettype[0]
+datatype[2].id -1466283082
+datatype[2].annotationreftype[0]
+datatype[2].arraytype[0]
+datatype[2].documenttype[0]
+datatype[2].structtype[1]
+datatype[2].structtype[0].name "annotation.person"
+datatype[2].structtype[0].version 0
+datatype[2].structtype[0].field[1]
+datatype[2].structtype[0].field[0].datatype 2
+datatype[2].structtype[0].field[0].name "name"
+datatype[2].structtype[0].field[0].id[0]
+datatype[2].structtype[0].inherits[0]
+datatype[2].weightedsettype[0]
+datatype[3].id -476092672
+datatype[3].annotationreftype[0]
+datatype[3].arraytype[0]
+datatype[3].documenttype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name "annotation.company"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[3]
+datatype[3].structtype[0].field[0].datatype 1184817987
+datatype[3].structtype[0].field[0].name "directors"
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[1].datatype 2
+datatype[3].structtype[0].field[1].name "name"
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].field[2].datatype 1321486441
+datatype[3].structtype[0].field[2].name "place"
+datatype[3].structtype[0].field[2].id[0]
+datatype[3].structtype[0].inherits[1]
+datatype[3].structtype[0].inherits[0].name "annotation.industry"
+datatype[3].structtype[0].inherits[0].version 0
+datatype[3].weightedsettype[0]
+datatype[4].id -945638949
+datatype[4].annotationreftype[0]
+datatype[4].arraytype[0]
+datatype[4].documenttype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name "blog.header"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[4]
+datatype[4].structtype[0].field[0].datatype 2
+datatype[4].structtype[0].field[0].name "author"
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[1].datatype 2
+datatype[4].structtype[0].field[1].name "body"
+datatype[4].structtype[0].field[1].id[0]
+datatype[4].structtype[0].field[2].datatype 2
+datatype[4].structtype[0].field[2].name "title"
+datatype[4].structtype[0].field[2].id[0]
+datatype[4].structtype[0].field[3].datatype 10
+datatype[4].structtype[0].field[3].name "url"
+datatype[4].structtype[0].field[3].id[0]
+datatype[4].structtype[0].inherits[0]
+datatype[4].weightedsettype[0]
+datatype[5].id 1184817987
+datatype[5].annotationreftype[0]
+datatype[5].arraytype[1]
+datatype[5].arraytype[0].datatype -1149562679
+datatype[5].documenttype[0]
+datatype[5].structtype[0]
+datatype[5].weightedsettype[0]
+datatype[6].id 1321486441
+datatype[6].annotationreftype[1]
+datatype[6].annotationreftype[0].annotation "location"
+datatype[6].arraytype[0]
+datatype[6].documenttype[0]
+datatype[6].structtype[0]
+datatype[6].weightedsettype[0]
+datatype[7].id 1387420336
+datatype[7].annotationreftype[0]
+datatype[7].arraytype[0]
+datatype[7].documenttype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name "blog.body"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[0]
+datatype[7].structtype[0].inherits[0]
+datatype[7].weightedsettype[0]
+datatype[8].id 515587158
+datatype[8].annotationreftype[0]
+datatype[8].arraytype[0]
+datatype[8].documenttype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name "annotation.location"
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[2]
+datatype[8].structtype[0].field[0].datatype 1
+datatype[8].structtype[0].field[0].name "lat"
+datatype[8].structtype[0].field[0].id[0]
+datatype[8].structtype[0].field[1].datatype 1
+datatype[8].structtype[0].field[1].name "lon"
+datatype[8].structtype[0].field[1].id[0]
+datatype[8].structtype[0].inherits[0]
+datatype[8].weightedsettype[0]
+datatype[9].id 912259135
+datatype[9].annotationreftype[0]
+datatype[9].arraytype[0]
+datatype[9].documenttype[0]
+datatype[9].structtype[1]
+datatype[9].structtype[0].name "annotation.industry"
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].field[1]
+datatype[9].structtype[0].field[0].datatype 2
+datatype[9].structtype[0].field[0].name "vertical"
+datatype[9].structtype[0].field[0].id[0]
+datatype[9].structtype[0].inherits[0]
+datatype[9].weightedsettype[0]
diff --git a/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4261985.cfg b/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4261985.cfg
new file mode 100644
index 00000000000..bbc67412652
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4261985.cfg
@@ -0,0 +1,181 @@
+enablecompression false
+annotationtype[5]
+annotationtype[0].datatype 1574163290
+annotationtype[0].id 1883143872
+annotationtype[0].name "bigshots"
+annotationtype[0].inherits[0]
+annotationtype[1].datatype -476092672
+annotationtype[1].id 1278713514
+annotationtype[1].name "company"
+annotationtype[1].inherits[1]
+annotationtype[1].inherits[0].id 9765800
+annotationtype[2].datatype 912259135
+annotationtype[2].id 9765800
+annotationtype[2].name "industry"
+annotationtype[2].inherits[0]
+annotationtype[3].datatype 515587158
+annotationtype[3].id -270471211
+annotationtype[3].name "location"
+annotationtype[3].inherits[0]
+annotationtype[4].datatype -1466283082
+annotationtype[4].id 609952424
+annotationtype[4].name "person"
+annotationtype[4].inherits[0]
+datatype[13]
+datatype[00].id -1149562679
+datatype[00].annotationreftype[1]
+datatype[00].annotationreftype[0].annotation "person"
+datatype[00].arraytype[0]
+datatype[00].documenttype[0]
+datatype[00].structtype[0]
+datatype[00].weightedsettype[0]
+datatype[01].id -1386162972
+datatype[01].annotationreftype[0]
+datatype[01].arraytype[0]
+datatype[01].documenttype[1]
+datatype[01].documenttype[0].bodystruct 1387420336
+datatype[01].documenttype[0].headerstruct -945638949
+datatype[01].documenttype[0].name "blog"
+datatype[01].documenttype[0].version 0
+datatype[01].documenttype[0].inherits[0]
+datatype[01].structtype[0]
+datatype[01].weightedsettype[0]
+datatype[02].id -1466283082
+datatype[02].annotationreftype[0]
+datatype[02].arraytype[0]
+datatype[02].documenttype[0]
+datatype[02].structtype[1]
+datatype[02].structtype[0].name "annotation.person"
+datatype[02].structtype[0].version 0
+datatype[02].structtype[0].field[1]
+datatype[02].structtype[0].field[0].datatype 2
+datatype[02].structtype[0].field[0].name "name"
+datatype[02].structtype[0].field[0].id[0]
+datatype[02].structtype[0].inherits[0]
+datatype[02].weightedsettype[0]
+datatype[03].id -476092672
+datatype[03].annotationreftype[0]
+datatype[03].arraytype[0]
+datatype[03].documenttype[0]
+datatype[03].structtype[1]
+datatype[03].structtype[0].name "annotation.company"
+datatype[03].structtype[0].version 0
+datatype[03].structtype[0].field[2]
+datatype[03].structtype[0].field[0].datatype 1184817987
+datatype[03].structtype[0].field[0].name "directors"
+datatype[03].structtype[0].field[0].id[0]
+datatype[03].structtype[0].field[1].datatype 2
+datatype[03].structtype[0].field[1].name "name"
+datatype[03].structtype[0].field[1].id[0]
+datatype[03].structtype[0].inherits[1]
+datatype[03].structtype[0].inherits[0].name "annotation.industry"
+datatype[03].structtype[0].inherits[0].version 0
+datatype[03].weightedsettype[0]
+datatype[04].id -945638949
+datatype[04].annotationreftype[0]
+datatype[04].arraytype[0]
+datatype[04].documenttype[0]
+datatype[04].structtype[1]
+datatype[04].structtype[0].name "blog.header"
+datatype[04].structtype[0].version 0
+datatype[04].structtype[0].field[4]
+datatype[04].structtype[0].field[0].datatype 2
+datatype[04].structtype[0].field[0].name "author"
+datatype[04].structtype[0].field[0].id[0]
+datatype[04].structtype[0].field[1].datatype 2
+datatype[04].structtype[0].field[1].name "body"
+datatype[04].structtype[0].field[1].id[0]
+datatype[04].structtype[0].field[2].datatype 2
+datatype[04].structtype[0].field[2].name "title"
+datatype[04].structtype[0].field[2].id[0]
+datatype[04].structtype[0].field[3].datatype 10
+datatype[04].structtype[0].field[3].name "url"
+datatype[04].structtype[0].field[3].id[0]
+datatype[04].structtype[0].inherits[0]
+datatype[04].weightedsettype[0]
+datatype[05].id 108708069
+datatype[05].annotationreftype[1]
+datatype[05].annotationreftype[0].annotation "bigshots"
+datatype[05].arraytype[0]
+datatype[05].documenttype[0]
+datatype[05].structtype[0]
+datatype[05].weightedsettype[0]
+datatype[06].id 1184817987
+datatype[06].annotationreftype[0]
+datatype[06].arraytype[1]
+datatype[06].arraytype[0].datatype -1149562679
+datatype[06].documenttype[0]
+datatype[06].structtype[0]
+datatype[06].weightedsettype[0]
+datatype[07].id 1321486441
+datatype[07].annotationreftype[1]
+datatype[07].annotationreftype[0].annotation "location"
+datatype[07].arraytype[0]
+datatype[07].documenttype[0]
+datatype[07].structtype[0]
+datatype[07].weightedsettype[0]
+datatype[08].id 1387420336
+datatype[08].annotationreftype[0]
+datatype[08].arraytype[0]
+datatype[08].documenttype[0]
+datatype[08].structtype[1]
+datatype[08].structtype[0].name "blog.body"
+datatype[08].structtype[0].version 0
+datatype[08].structtype[0].field[0]
+datatype[08].structtype[0].inherits[0]
+datatype[08].weightedsettype[0]
+datatype[09].id 1574163290
+datatype[09].annotationreftype[0]
+datatype[09].arraytype[0]
+datatype[09].documenttype[0]
+datatype[09].structtype[1]
+datatype[09].structtype[0].name "annotation.bigshots"
+datatype[09].structtype[0].version 0
+datatype[09].structtype[0].field[2]
+datatype[09].structtype[0].field[0].datatype 1975335457
+datatype[09].structtype[0].field[0].name "ceos"
+datatype[09].structtype[0].field[0].id[0]
+datatype[09].structtype[0].field[1].datatype 108708069
+datatype[09].structtype[0].field[1].name "self"
+datatype[09].structtype[0].field[1].id[0]
+datatype[09].structtype[0].inherits[0]
+datatype[09].weightedsettype[0]
+datatype[10].id 1975335457
+datatype[10].annotationreftype[1]
+datatype[10].annotationreftype[0].annotation "company"
+datatype[10].arraytype[0]
+datatype[10].documenttype[0]
+datatype[10].structtype[0]
+datatype[10].weightedsettype[0]
+datatype[11].id 515587158
+datatype[11].annotationreftype[0]
+datatype[11].arraytype[0]
+datatype[11].documenttype[0]
+datatype[11].structtype[1]
+datatype[11].structtype[0].name "annotation.location"
+datatype[11].structtype[0].version 0
+datatype[11].structtype[0].field[2]
+datatype[11].structtype[0].field[0].datatype 1
+datatype[11].structtype[0].field[0].name "lat"
+datatype[11].structtype[0].field[0].id[0]
+datatype[11].structtype[0].field[1].datatype 1
+datatype[11].structtype[0].field[1].name "lon"
+datatype[11].structtype[0].field[1].id[0]
+datatype[11].structtype[0].inherits[0]
+datatype[11].weightedsettype[0]
+datatype[12].id 912259135
+datatype[12].annotationreftype[0]
+datatype[12].arraytype[0]
+datatype[12].documenttype[0]
+datatype[12].structtype[1]
+datatype[12].structtype[0].name "annotation.industry"
+datatype[12].structtype[0].version 0
+datatype[12].structtype[0].field[2]
+datatype[12].structtype[0].field[0].datatype 1321486441
+datatype[12].structtype[0].field[0].name "place"
+datatype[12].structtype[0].field[0].id[0]
+datatype[12].structtype[0].field[1].datatype 2
+datatype[12].structtype[0].field[1].name "vertical"
+datatype[12].structtype[0].field[1].id[0]
+datatype[12].structtype[0].inherits[0]
+datatype[12].weightedsettype[0]
diff --git a/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4475379.cfg b/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4475379.cfg
new file mode 100644
index 00000000000..ae299ab81fa
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/documentmanager.bug4475379.cfg
@@ -0,0 +1,129 @@
+enablecompression false
+annotationtype[4]
+annotationtype[0].datatype -476092672
+annotationtype[0].id 1278713514
+annotationtype[0].name "company"
+annotationtype[0].inherits[1]
+annotationtype[0].inherits[0].id 9765800
+annotationtype[1].datatype 912259135
+annotationtype[1].id 9765800
+annotationtype[1].name "industry"
+annotationtype[1].inherits[0]
+annotationtype[2].datatype 515587158
+annotationtype[2].id -270471211
+annotationtype[2].name "location"
+annotationtype[2].inherits[0]
+annotationtype[3].datatype -1466283082
+annotationtype[3].id 609952424
+annotationtype[3].name "person"
+annotationtype[3].inherits[0]
+datatype[7]
+datatype[0].id -1386162972
+datatype[0].annotationreftype[0]
+datatype[0].arraytype[0]
+datatype[0].documenttype[1]
+datatype[0].documenttype[0].bodystruct 1387420336
+datatype[0].documenttype[0].headerstruct -945638949
+datatype[0].documenttype[0].name "blog"
+datatype[0].documenttype[0].version 0
+datatype[0].documenttype[0].inherits[0]
+datatype[0].structtype[0]
+datatype[0].weightedsettype[0]
+datatype[1].id -1466283082
+datatype[1].annotationreftype[0]
+datatype[1].arraytype[0]
+datatype[1].documenttype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name "annotation.person"
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[1]
+datatype[1].structtype[0].field[0].datatype 2
+datatype[1].structtype[0].field[0].name "name"
+datatype[1].structtype[0].field[0].id[0]
+datatype[1].structtype[0].inherits[0]
+datatype[1].weightedsettype[0]
+datatype[2].id -476092672
+datatype[2].annotationreftype[0]
+datatype[2].arraytype[0]
+datatype[2].documenttype[0]
+datatype[2].structtype[1]
+datatype[2].structtype[0].name "annotation.company"
+datatype[2].structtype[0].version 0
+datatype[2].structtype[0].field[5]
+datatype[2].structtype[0].field[0].datatype 4
+datatype[2].structtype[0].field[0].name "alt"
+datatype[2].structtype[0].field[0].id[0]
+datatype[2].structtype[0].field[1].datatype 2
+datatype[2].structtype[0].field[1].name "ceo"
+datatype[2].structtype[0].field[1].id[0]
+datatype[2].structtype[0].field[2].datatype 1
+datatype[2].structtype[0].field[2].name "lat"
+datatype[2].structtype[0].field[2].id[0]
+datatype[2].structtype[0].field[3].datatype 1
+datatype[2].structtype[0].field[3].name "lon"
+datatype[2].structtype[0].field[3].id[0]
+datatype[2].structtype[0].field[4].datatype 2
+datatype[2].structtype[0].field[4].name "name"
+datatype[2].structtype[0].field[4].id[0]
+datatype[2].structtype[0].inherits[1]
+datatype[2].structtype[0].inherits[0].name "annotation.industry"
+datatype[2].structtype[0].inherits[0].version 0
+datatype[2].weightedsettype[0]
+datatype[3].id -945638949
+datatype[3].annotationreftype[0]
+datatype[3].arraytype[0]
+datatype[3].documenttype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name "blog.header"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[4]
+datatype[3].structtype[0].field[0].datatype 2
+datatype[3].structtype[0].field[0].name "author"
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[1].datatype 2
+datatype[3].structtype[0].field[1].name "body"
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].field[2].datatype 2
+datatype[3].structtype[0].field[2].name "title"
+datatype[3].structtype[0].field[2].id[0]
+datatype[3].structtype[0].field[3].datatype 10
+datatype[3].structtype[0].field[3].name "url"
+datatype[3].structtype[0].field[3].id[0]
+datatype[3].structtype[0].inherits[0]
+datatype[3].weightedsettype[0]
+datatype[4].id 1387420336
+datatype[4].annotationreftype[0]
+datatype[4].arraytype[0]
+datatype[4].documenttype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name "blog.body"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[0]
+datatype[4].structtype[0].inherits[0]
+datatype[4].weightedsettype[0]
+datatype[5].id 515587158
+datatype[5].annotationreftype[0]
+datatype[5].arraytype[0]
+datatype[5].documenttype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name "annotation.location"
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].field[1]
+datatype[5].structtype[0].field[0].datatype 2
+datatype[5].structtype[0].field[0].name "name"
+datatype[5].structtype[0].field[0].id[0]
+datatype[5].structtype[0].inherits[0]
+datatype[5].weightedsettype[0]
+datatype[6].id 912259135
+datatype[6].annotationreftype[0]
+datatype[6].arraytype[0]
+datatype[6].documenttype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name "annotation.industry"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[1]
+datatype[6].structtype[0].field[0].datatype 2
+datatype[6].structtype[0].field[0].name "vertical"
+datatype[6].structtype[0].field[0].id[0]
+datatype[6].structtype[0].inherits[0]
+datatype[6].weightedsettype[0]
diff --git a/document/src/test/java/com/yahoo/document/annotation/documentmanager.systemtest.cfg b/document/src/test/java/com/yahoo/document/annotation/documentmanager.systemtest.cfg
new file mode 100644
index 00000000000..ce8be4410a1
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/annotation/documentmanager.systemtest.cfg
@@ -0,0 +1,155 @@
+enablecompression false
+datatype[11]
+datatype[0].id -198681903
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[0].annotationreftype[1]
+datatype[0].annotationreftype[0].annotation "person"
+datatype[1].id 1812054936
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[1].annotationreftype[1]
+datatype[1].annotationreftype[0].annotation "date"
+datatype[2].id 692270031
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[2].annotationreftype[1]
+datatype[2].annotationreftype[0].annotation "place"
+datatype[3].id 892457735
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name "article.header"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[2]
+datatype[3].structtype[0].field[0].name "title"
+datatype[3].structtype[0].field[0].datatype 2
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[1].name "content"
+datatype[3].structtype[0].field[1].datatype 2
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].inherits[0]
+datatype[3].documenttype[0]
+datatype[3].annotationreftype[0]
+datatype[4].id -1984964900
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name "article.body"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[0]
+datatype[4].structtype[0].inherits[0]
+datatype[4].documenttype[0]
+datatype[4].annotationreftype[0]
+datatype[5].id 559508792
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[0]
+datatype[5].documenttype[1]
+datatype[5].documenttype[0].name "article"
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].headerstruct 892457735
+datatype[5].documenttype[0].bodystruct -1984964900
+datatype[5].documenttype[0].inherits[0]
+datatype[5].annotationreftype[0]
+datatype[6].id -1466283082
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name "annotation.person"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[1]
+datatype[6].structtype[0].field[0].name "name"
+datatype[6].structtype[0].field[0].datatype 2
+datatype[6].structtype[0].field[0].id[0]
+datatype[6].structtype[0].inherits[0]
+datatype[6].documenttype[0]
+datatype[6].annotationreftype[0]
+datatype[7].id 1157126952
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name "annotation.artist"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[1]
+datatype[7].structtype[0].field[0].name "instrument"
+datatype[7].structtype[0].field[0].datatype 0
+datatype[7].structtype[0].field[0].id[0]
+datatype[7].structtype[0].inherits[1]
+datatype[7].structtype[0].inherits[0].name "annotation.person"
+datatype[7].structtype[0].inherits[0].version 0
+datatype[7].documenttype[0]
+datatype[7].annotationreftype[0]
+datatype[8].id -840345201
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name "annotation.date"
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[1]
+datatype[8].structtype[0].field[0].name "exacttime"
+datatype[8].structtype[0].field[0].datatype 4
+datatype[8].structtype[0].field[0].id[0]
+datatype[8].structtype[0].inherits[0]
+datatype[8].documenttype[0]
+datatype[8].annotationreftype[0]
+datatype[9].id 2076579146
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[1]
+datatype[9].structtype[0].name "annotation.place"
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].field[2]
+datatype[9].structtype[0].field[0].name "lat"
+datatype[9].structtype[0].field[0].datatype 4
+datatype[9].structtype[0].field[0].id[0]
+datatype[9].structtype[0].field[1].name "lon"
+datatype[9].structtype[0].field[1].datatype 4
+datatype[9].structtype[0].field[1].id[0]
+datatype[9].structtype[0].inherits[0]
+datatype[9].documenttype[0]
+datatype[9].annotationreftype[0]
+datatype[10].id 1194300957
+datatype[10].arraytype[0]
+datatype[10].weightedsettype[0]
+datatype[10].structtype[1]
+datatype[10].structtype[0].name "annotation.event"
+datatype[10].structtype[0].version 0
+datatype[10].structtype[0].field[4]
+datatype[10].structtype[0].field[0].name "description"
+datatype[10].structtype[0].field[0].datatype 2
+datatype[10].structtype[0].field[0].id[0]
+datatype[10].structtype[0].field[1].name "person"
+datatype[10].structtype[0].field[1].datatype -198681903
+datatype[10].structtype[0].field[1].id[0]
+datatype[10].structtype[0].field[2].name "date"
+datatype[10].structtype[0].field[2].datatype 1812054936
+datatype[10].structtype[0].field[2].id[0]
+datatype[10].structtype[0].field[3].name "place"
+datatype[10].structtype[0].field[3].datatype 692270031
+datatype[10].structtype[0].field[3].id[0]
+datatype[10].structtype[0].inherits[0]
+datatype[10].documenttype[0]
+datatype[10].annotationreftype[0]
+annotationtype[5]
+annotationtype[0].name "person"
+annotationtype[0].id 609952424
+annotationtype[0].datatype -1466283082
+annotationtype[1].name "event"
+annotationtype[1].id -455530995
+annotationtype[1].datatype 1194300957
+annotationtype[2].name "artist"
+annotationtype[2].id 690330276
+annotationtype[2].datatype 1157126952
+annotationtype[3].name "date"
+annotationtype[3].id -162455681
+annotationtype[3].datatype -840345201
+annotationtype[4].name "place"
+annotationtype[4].id 1707984040
+annotationtype[4].datatype 2076579146
diff --git a/document/src/test/java/com/yahoo/document/datatypes/ArrayTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/ArrayTestCase.java
new file mode 100755
index 00000000000..a2f0581f003
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/ArrayTestCase.java
@@ -0,0 +1,258 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+
+import java.util.*;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class ArrayTestCase extends junit.framework.TestCase {
+    public void testToArray() {
+        ArrayDataType dt = new ArrayDataType(DataType.STRING);
+        Array<StringFieldValue> arr = new Array<>(dt);
+        arr.add(new StringFieldValue("a"));
+        arr.add(new StringFieldValue("b"));
+        arr.add(new StringFieldValue("c"));
+        StringFieldValue[] tooSmall = new StringFieldValue[0];
+        StringFieldValue[] bigEnough = new StringFieldValue[3];
+        StringFieldValue[] a = arr.toArray(tooSmall);
+        assertNotSame(tooSmall, a);
+        assertEquals(new StringFieldValue("a"), a[0]);
+        assertEquals(new StringFieldValue("b"), a[1]);
+        assertEquals(new StringFieldValue("c"), a[2]);
+        StringFieldValue[] b = arr.toArray(bigEnough);
+        assertSame(bigEnough, b);
+        assertEquals(new StringFieldValue("a"), b[0]);
+        assertEquals(new StringFieldValue("b"), b[1]);
+        assertEquals(new StringFieldValue("c"), b[2]);
+    }
+
+    public void testCreateIllegalArray() {
+        ArrayList<FieldValue> arrayList = new ArrayList<>();
+        arrayList.add(new StringFieldValue("foo"));
+        arrayList.add(new IntegerFieldValue(1000));
+        DataType stringType = new ArrayDataType(DataType.STRING);
+        try {
+            Array<FieldValue> illegalArray = new Array<>(stringType, arrayList);
+            fail("Expected an exception");
+        } catch (IllegalArgumentException e) {
+            assertEquals("FieldValue 1000 is not compatible with datatype "
+                         + "Array<string> (code: -1486737430).",
+                         e.getMessage());
+        }
+
+        DataType intType = new ArrayDataType(DataType.INT);
+        Array<IntegerFieldValue> intArray = new Array<>(intType);
+        intArray.add(new IntegerFieldValue(42));
+        Array<StringFieldValue> stringArray = new Array<>(stringType);
+        try {
+            stringArray.assign(intArray);
+            fail("Expected an exception");
+        } catch (IllegalArgumentException e) {
+            assertEquals("Incompatible data types. Got datatype int (code: 0),"
+                         + " expected datatype string (code: 2)",
+                         e.getMessage());
+        }
+    }
+
+    public void testWrappedList() {
+        Array<StringFieldValue> array = new Array<StringFieldValue>(DataType.getArray(DataType.STRING));
+        List<String> list = new ArrayList<>();
+        list.add("foo");
+        list.add("bar");
+        list.add("baz");
+
+        array.assign(list);
+
+        assertEquals(3, array.size());
+        assertEquals(3, list.size());
+        assertFalse(array.isEmpty());
+        assertFalse(list.isEmpty());
+        assertEquals("foo", list.get(0));
+        assertEquals("bar", list.get(1));
+        assertEquals("baz", list.get(2));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("bar"), array.get(1));
+        assertEquals(new StringFieldValue("baz"), array.get(2));
+
+        array.remove(2);
+
+        assertEquals(2, array.size());
+        assertEquals(2, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("bar", list.get(1));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("bar"), array.get(1));
+
+        list.remove(1);
+
+        assertEquals(1, array.size());
+        assertEquals(1, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+
+
+        list.add("bar");
+
+        assertEquals(2, array.size());
+        assertEquals(2, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("bar", list.get(1));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("bar"), array.get(1));
+
+        array.add(new StringFieldValue("baz"));
+
+        assertEquals(3, array.size());
+        assertEquals(3, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("bar", list.get(1));
+        assertEquals("baz", list.get(2));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("bar"), array.get(1));
+        assertEquals(new StringFieldValue("baz"), array.get(2));
+
+        assertTrue(array.contains(new StringFieldValue("foo")));
+        assertTrue(list.contains("foo"));
+
+        list.add("foo");
+
+        assertEquals(0, list.indexOf("foo"));
+        assertEquals(0, array.indexOf(new StringFieldValue("foo")));
+        assertEquals(3, list.lastIndexOf("foo"));
+        assertEquals(3, array.lastIndexOf(new StringFieldValue("foo")));
+
+        list.set(3, "banana");
+
+        assertEquals(4, array.size());
+        assertEquals(4, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("bar", list.get(1));
+        assertEquals("baz", list.get(2));
+        assertEquals("banana", list.get(3));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("bar"), array.get(1));
+        assertEquals(new StringFieldValue("baz"), array.get(2));
+        assertEquals(new StringFieldValue("banana"), array.get(3));
+
+        array.set(3, new StringFieldValue("apple"));
+
+        assertEquals(4, array.size());
+        assertEquals(4, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("bar", list.get(1));
+        assertEquals("baz", list.get(2));
+        assertEquals("apple", list.get(3));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("bar"), array.get(1));
+        assertEquals(new StringFieldValue("baz"), array.get(2));
+        assertEquals(new StringFieldValue("apple"), array.get(3));
+
+        list.remove("bar");
+
+        assertEquals(3, array.size());
+        assertEquals(3, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("baz", list.get(1));
+        assertEquals("apple", list.get(2));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("baz"), array.get(1));
+        assertEquals(new StringFieldValue("apple"), array.get(2));
+
+        array.remove(new StringFieldValue("baz"));
+
+        assertEquals(2, array.size());
+        assertEquals(2, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("apple", list.get(1));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("apple"), array.get(1));
+
+        assertNotNull(array.toArray(new StringFieldValue[5]));
+
+        try {
+            array.retainAll(new ArrayList<StringFieldValue>());
+            fail("Not implemented yet.");
+        } catch (UnsupportedOperationException uoe) {
+            //ok!
+        }
+
+        array.add(1, new StringFieldValue("boo"));
+
+        assertEquals(3, array.size());
+        assertEquals(3, list.size());
+        assertEquals("foo", list.get(0));
+        assertEquals("boo", list.get(1));
+        assertEquals("apple", list.get(2));
+        assertEquals(new StringFieldValue("foo"), array.get(0));
+        assertEquals(new StringFieldValue("boo"), array.get(1));
+        assertEquals(new StringFieldValue("apple"), array.get(2));
+
+        array.toString();
+
+        List<StringFieldValue> subArray = array.subList(1, 3);
+        assertEquals(2, subArray.size());
+        assertEquals(new StringFieldValue("boo"), subArray.get(0));
+        assertEquals(new StringFieldValue("apple"), subArray.get(1));
+
+
+        assertEquals(false, array.containsAll(Arrays.<StringFieldValue>asList(new StringFieldValue("bob"))));
+        assertEquals(true, array.containsAll(Arrays.<StringFieldValue>asList(new StringFieldValue("foo"), new StringFieldValue("boo"), new StringFieldValue("apple"))));
+
+        array.removeAll(Arrays.<StringFieldValue>asList(new StringFieldValue("foo"), new StringFieldValue("boo")));
+
+        assertEquals(1, array.size());
+        assertEquals(1, list.size());
+        assertEquals("apple", list.get(0));
+        assertEquals(new StringFieldValue("apple"), array.get(0));
+
+        array.add(new StringFieldValue("ibm"));
+
+        assertEquals(2, array.size());
+        assertEquals(2, list.size());
+
+        {
+            Iterator<StringFieldValue> it = array.iterator();
+            assertTrue(it.hasNext());
+            assertEquals(new StringFieldValue("apple"), it.next());
+            assertTrue(it.hasNext());
+            assertEquals(new StringFieldValue("ibm"), it.next());
+            assertFalse(it.hasNext());
+        }
+        {
+            ListIterator<StringFieldValue> it = array.listIterator();
+            assertTrue(it.hasNext());
+            assertEquals(new StringFieldValue("apple"), it.next());
+            assertTrue(it.hasNext());
+            assertEquals(new StringFieldValue("ibm"), it.next());
+            assertFalse(it.hasNext());
+        }
+
+        array.addAll(Arrays.<StringFieldValue>asList(new StringFieldValue("microsoft"), new StringFieldValue("google")));
+
+        assertEquals(4, array.size());
+        assertEquals(4, list.size());
+
+        array.clear();
+
+        assertEquals(0, array.size());
+        assertEquals(0, list.size());
+
+    }
+
+    public void testListWrapperToArray() {
+        Array<StringFieldValue> array = new Array<>(new ArrayDataType(DataType.STRING));
+        List<StringFieldValue> assignFrom = new ArrayList<>(3);
+        assignFrom.add(new StringFieldValue("a"));
+        assignFrom.add(new StringFieldValue("b"));
+        assignFrom.add(new StringFieldValue("c"));
+        array.assign(assignFrom);
+        final StringFieldValue[] expected = new StringFieldValue[] { new StringFieldValue("a"), new StringFieldValue("b"),
+                new StringFieldValue("c") };
+        assertTrue(Arrays.equals(expected, array.toArray(new StringFieldValue[0])));
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/MapTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/MapTestCase.java
new file mode 100644
index 00000000000..0b15ccf9854
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/MapTestCase.java
@@ -0,0 +1,166 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import java.util.Map;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.Field;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+
+
+public class MapTestCase extends junit.framework.TestCase {
+
+    public void testStringMap() {
+        MapDataType mapType = new MapDataType(DataType.STRING, DataType.STRING);
+        MapFieldValue<StringFieldValue, StringFieldValue> map = mapType.createFieldValue();
+        StringFieldValue sfvk1=new StringFieldValue("k1");
+        StringFieldValue sfvk2=new StringFieldValue("k2");
+        StringFieldValue sfvk3=new StringFieldValue("k3");
+        StringFieldValue sfvv1=new StringFieldValue("v1");
+        StringFieldValue sfvv2=new StringFieldValue("v2");
+        StringFieldValue sfvv3=new StringFieldValue("v3");
+        map.put(sfvk1, sfvv1);
+        map.put(sfvk2, sfvv2);
+        map.put(sfvk3, sfvv3);
+        assertEquals(map.get(sfvk1), sfvv1);
+        assertEquals(map.get(sfvk2), sfvv2);
+        assertEquals(map.get(sfvk3), sfvv3);
+        assertEquals(map.get(new StringFieldValue("k1")).getWrappedValue(), "v1");
+        assertEquals(map.get(new StringFieldValue("k2")).getWrappedValue(), "v2");
+        assertEquals(map.get(new StringFieldValue("k3")).getWrappedValue(), "v3");
+    }
+
+    public void testAdvancedMap() {
+        MapDataType stringMapType1 = new MapDataType(DataType.STRING, DataType.STRING);
+        MapDataType stringMapType2 = new MapDataType(DataType.STRING, DataType.STRING);
+        MapFieldValue sm1 = stringMapType1.createFieldValue();
+        MapFieldValue sm2 = stringMapType2.createFieldValue();
+        StringFieldValue e = new StringFieldValue("e");
+        StringFieldValue g = new StringFieldValue("g");
+        sm1.put(new StringFieldValue("a"), new StringFieldValue("b"));
+        sm1.put(new StringFieldValue("c"), new StringFieldValue("d"));
+        sm2.put(e, new StringFieldValue("f"));
+        sm2.put(g, new StringFieldValue("h"));
+
+        StructDataType structType= new StructDataType("teststr");
+        structType.addField(new Field("int", DataType.INT));
+        structType.addField(new Field("flt", DataType.FLOAT));
+        Struct s = structType.createFieldValue();
+        s.setFieldValue("int", 99);
+        s.setFieldValue("flt", -89.345);
+
+        ArrayDataType twoDimArray = DataType.getArray(DataType.getArray(DataType.FLOAT));
+        Array tda = twoDimArray.createFieldValue();
+
+        MapDataType floatToTwoDimArray = new MapDataType(DataType.FLOAT, twoDimArray);
+        MapDataType stringToStruct = new MapDataType(DataType.STRING, structType);
+        MapDataType stringMapToStringMap = new MapDataType(stringMapType1, stringMapType2);
+
+        MapFieldValue f2tda = floatToTwoDimArray.createFieldValue();
+        f2tda.put(new FloatFieldValue(3.4f), tda);
+
+        MapFieldValue s2sct = stringToStruct.createFieldValue();
+        s2sct.put(new StringFieldValue("s1"), s);
+        MapFieldValue sm2sm = stringMapToStringMap.createFieldValue();
+        sm2sm.put(sm1, sm2);
+
+        assertEquals(f2tda.get(new FloatFieldValue(3.4f)), tda);
+        assertEquals(new IntegerFieldValue(99), ((Struct) (s2sct.get(new StringFieldValue("s1")))).getFieldValue("int"));
+        assertEquals(new StringFieldValue("f"), ((MapFieldValue)(sm2sm.get(sm1))).get(e));
+        assertEquals(new StringFieldValue("h"), ((MapFieldValue)(sm2sm.get(sm1))).get(g));
+
+        // Look up using different map w same contents
+        // TODO it works even if sm1_2 is empty, something with class id?
+        MapFieldValue sm1_2 = stringMapType1.createFieldValue();
+        sm1_2.put(new StringFieldValue("a"), new StringFieldValue("b"));
+        sm1_2.put(new StringFieldValue("c"), new StringFieldValue("d"));
+        assertEquals(new StringFieldValue ("f"), ((MapFieldValue)(sm2sm.get(sm1_2))).get(e));
+        assertEquals(new StringFieldValue("h"), ((MapFieldValue)(sm2sm.get(sm1_2))).get(g));
+
+    }
+
+    public void testSerializationStringMap() {
+        MapDataType mapType = new MapDataType(DataType.STRING, DataType.STRING);
+        MapFieldValue<StringFieldValue, StringFieldValue> map = mapType.createFieldValue();
+        //Field f = new Field("stringmap",mapType);
+        StringFieldValue sfvk1=new StringFieldValue("k1");
+        StringFieldValue sfvk2=new StringFieldValue("k2");
+        StringFieldValue sfvk3=new StringFieldValue("k3");
+        StringFieldValue sfvv1=new StringFieldValue("v1");
+        StringFieldValue sfvv2=new StringFieldValue("v2");
+        StringFieldValue sfvv3=new StringFieldValue("v3");
+        map.put(sfvk1, sfvv1);
+        map.put(sfvk2, sfvv2);
+        map.put(sfvk3, sfvv3);
+        assertCorrectSerialization(mapType, map);
+    }
+
+    public void testSerializationComplex() {
+        ArrayDataType twoDimArray = DataType.getArray(DataType.getArray(DataType.FLOAT));
+        MapDataType floatToTwoDimArray = new MapDataType(DataType.FLOAT, twoDimArray);
+        MapFieldValue<FloatFieldValue, Array<Array<FloatFieldValue>>> map = floatToTwoDimArray.createFieldValue();
+
+        Array<FloatFieldValue> af1 = new Array(DataType.getArray(DataType.FLOAT));
+        af1.add(new FloatFieldValue(11f));
+        af1.add(new FloatFieldValue(111f));
+        Array<FloatFieldValue> af2 = new Array(DataType.getArray(DataType.FLOAT));
+        af2.add(new FloatFieldValue(22f));
+        af2.add(new FloatFieldValue(222f));
+        Array<Array<FloatFieldValue>> aaf1 = new Array(twoDimArray);
+        aaf1.add(af1);
+        aaf1.add(af2);
+
+        Array<FloatFieldValue> af3 = new Array(DataType.getArray(DataType.FLOAT));
+        af3.add(new FloatFieldValue(33f));
+        af3.add(new FloatFieldValue(333f));
+        Array<FloatFieldValue> af4 = new Array(DataType.getArray(DataType.FLOAT));
+        af4.add(new FloatFieldValue(44f));
+        af4.add(new FloatFieldValue(444f));
+        Array<Array<FloatFieldValue>> aaf2 = new Array(twoDimArray);
+        aaf2.add(af3);
+        aaf2.add(af4);
+
+        map.put(new FloatFieldValue(1.1f), aaf1);
+        map.put(new FloatFieldValue(2.2f), aaf2);
+        assertCorrectSerialization(floatToTwoDimArray, map);
+    }
+
+    private void assertCorrectSerialization(MapDataType mapType, MapFieldValue<? extends FieldValue, ? extends FieldValue> map) {
+        Field f = new Field("", mapType);
+        DocumentTypeManager man = new DocumentTypeManager();
+        man.register(mapType);
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+        serializer.write(f, map);
+        buffer.flip();
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+        MapFieldValue<FieldValue, FieldValue> map2 = new MapFieldValue<FieldValue, FieldValue>(mapType);
+        deserializer.read(f, map2);
+        assertNotSame(map, map2);
+        for (Map.Entry<?,?> e : map.entrySet()) {
+            assertEquals(e.getValue(), map2.get(e.getKey()));
+        }
+    }
+
+    public void testIllegalMapAssignment() {
+        MapDataType type1 = new MapDataType(DataType.INT, DataType.INT);
+        MapDataType type2 = new MapDataType(DataType.STRING, DataType.STRING);
+        MapFieldValue map1 = new MapFieldValue(type1);
+        map1.put(new IntegerFieldValue(42), new IntegerFieldValue(84));
+        MapFieldValue map2 = new MapFieldValue(type2);
+        try {
+            map2.assign(map1);
+            fail("Expected an exception");
+        } catch (IllegalArgumentException e) {
+            assertEquals("Incompatible data types. Got datatype int (code: 0),"
+                         + " expected datatype string (code: 2)",
+                         e.getMessage());
+        }
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/NumericFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/NumericFieldValueTestCase.java
new file mode 100644
index 00000000000..c7f88ac64e9
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/NumericFieldValueTestCase.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * Created by balder on 8/6/15.
+ */
+public class NumericFieldValueTestCase {
+    @Test
+    public void requireThatConstructionFromStringWorks() {
+        assertEquals(67, new IntegerFieldValue("67").getInteger());
+        assertEquals(67, new LongFieldValue("67").getLong());
+        assertEquals(67, new ByteFieldValue("67").getByte());
+        assertEquals(67.45, new FloatFieldValue("67.45").getFloat(),    0.00001);
+        assertEquals(67.45, new DoubleFieldValue("67.45").getDouble(), 0.000001);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/PredicateFieldValueTest.java b/document/src/test/java/com/yahoo/document/datatypes/PredicateFieldValueTest.java
new file mode 100644
index 00000000000..b2b1fde0126
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/PredicateFieldValueTest.java
@@ -0,0 +1,193 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.predicate.FeatureSet;
+import com.yahoo.document.predicate.SimplePredicates;
+import com.yahoo.document.predicate.Predicate;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlStream;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class PredicateFieldValueTest {
+
+    @Test
+    public void requireThatFactoryProducesPredicate() {
+        assertEquals(PredicateFieldValue.class, PredicateFieldValue.getFactory().create().getClass());
+    }
+
+    @Test
+    public void requireThatAccessorsWork() {
+        PredicateFieldValue value = new PredicateFieldValue();
+        Predicate predicate = SimplePredicates.newPredicate();
+        value.setPredicate(predicate);
+        assertSame(predicate, value.getPredicate());
+    }
+
+    @Test
+    public void requireThatConstructorsWork() {
+        assertNull(new PredicateFieldValue().getPredicate());
+
+        Predicate predicate = SimplePredicates.newPredicate();
+        assertEquals(predicate, new PredicateFieldValue(predicate).getPredicate());
+    }
+
+    @Test
+    public void requireThatDataTypeIsPredicate() {
+        assertEquals(DataType.PREDICATE,
+                     new PredicateFieldValue().getDataType());
+    }
+
+    @Test
+    public void requireThatXmlOutputIsEmptyForNullPredicate() {
+        XmlStream expected = new XmlStream();
+        expected.beginTag("tag");
+        expected.endTag();
+        assertEquals(expected.toString(), printXml("tag", new PredicateFieldValue()));
+    }
+
+    @Test
+    public void requireThatXmlOutputIsPredicateLanguage() {
+        Predicate predicate = new FeatureSet("key", "valueA", "valueB");
+        XmlStream expected = new XmlStream();
+        expected.beginTag("tag");
+        expected.addContent(predicate.toString());
+        expected.endTag();
+        assertEquals(expected.toString(), printXml("tag", new PredicateFieldValue(predicate)));
+    }
+
+    @Test
+    public void requireThatClearNullsPredicate() {
+        PredicateFieldValue value = new PredicateFieldValue(SimplePredicates.newPredicate());
+        value.clear();
+        assertNull(value.getPredicate());
+    }
+
+    @Test
+    public void requireThatNullCanBeAssigned() {
+        PredicateFieldValue value = new PredicateFieldValue(SimplePredicates.newPredicate());
+        value.assign(null);
+        assertNull(value.getPredicate());
+    }
+
+    @Test
+    public void requireThatPredicateCanBeAssigned() {
+        Predicate predicate = SimplePredicates.newPredicate();
+        PredicateFieldValue value = new PredicateFieldValue();
+        value.assign(predicate);
+        assertSame(predicate, value.getPredicate());
+    }
+
+    @Test
+    public void requireThatPredicateFieldValueCanBeAssigned() {
+        Predicate predicate = SimplePredicates.newPredicate();
+        PredicateFieldValue value = new PredicateFieldValue();
+        value.assign(new PredicateFieldValue(predicate));
+        assertSame(predicate, value.getPredicate());
+    }
+
+    @Test
+    public void requireThatBadAssignThrows() {
+        try {
+            new PredicateFieldValue().assign(new Object());
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Expected com.yahoo.document.datatypes.PredicateFieldValue, got java.lang.Object.",
+                         e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatSerializeCallsBackToWriter() {
+        Field field = Mockito.mock(Field.class);
+        FieldWriter writer = Mockito.mock(FieldWriter.class);
+        PredicateFieldValue value = new PredicateFieldValue(SimplePredicates.newPredicate());
+        value.serialize(field, writer);
+        Mockito.verify(writer).write(field, value);
+    }
+
+    @Test
+    public void requireThatDeserializeCallsBackToReader() {
+        Field field = Mockito.mock(Field.class);
+        FieldReader reader = Mockito.mock(FieldReader.class);
+        PredicateFieldValue value = new PredicateFieldValue(SimplePredicates.newPredicate());
+        value.deserialize(field, reader);
+        Mockito.verify(reader).read(field, value);
+    }
+
+    @Test
+    public void requireThatWrappedValueIsPredicate() {
+        Predicate predicate = SimplePredicates.newPredicate();
+        PredicateFieldValue value = new PredicateFieldValue(predicate);
+        assertSame(predicate, value.getWrappedValue());
+    }
+
+    @Test
+    public void requireThatCloneIsImplemented() {
+        PredicateFieldValue value1 = new PredicateFieldValue();
+        PredicateFieldValue value2 = value1.clone();
+        assertEquals(value1, value2);
+        assertNotSame(value1, value2);
+
+        value1 = new PredicateFieldValue(SimplePredicates.newString("foo"));
+        value2 = value1.clone();
+        assertEquals(value1, value2);
+        assertNotSame(value1, value2);
+        assertNotSame(value1.getPredicate(), value2.getPredicate());
+    }
+
+    @Test
+    public void requireThatHashCodeIsImplemented() {
+        assertEquals(new PredicateFieldValue().hashCode(),
+                     new PredicateFieldValue().hashCode());
+    }
+
+    @Test
+    public void requireThatEqualsIsImplemented() {
+        // null predicate in lhs
+        PredicateFieldValue lhs = new PredicateFieldValue();
+        assertTrue(lhs.equals(lhs));
+        assertFalse(lhs.equals(new Object()));
+
+        PredicateFieldValue rhs = new PredicateFieldValue(SimplePredicates.newString("bar"));
+        assertFalse(lhs.equals(rhs));
+        rhs.setPredicate(null);
+        assertTrue(lhs.equals(rhs));
+
+        // non-null predicate in lhs
+        lhs = new PredicateFieldValue(SimplePredicates.newString("foo"));
+        assertTrue(lhs.equals(lhs));
+        assertFalse(lhs.equals(new Object()));
+
+        rhs = new PredicateFieldValue();
+        assertFalse(lhs.equals(rhs));
+        rhs.setPredicate(SimplePredicates.newString("bar"));
+        assertFalse(lhs.equals(rhs));
+        rhs.setPredicate(SimplePredicates.newString("foo"));
+        assertTrue(lhs.equals(rhs));
+    }
+
+    private static String printXml(String tag, FieldValue value) {
+        XmlStream out = new XmlStream();
+        out.beginTag(tag);
+        if (value != null) {
+            value.printXml(out);
+        }
+        out.endTag();
+        return out.toString();
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/RawTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/RawTestCase.java
new file mode 100644
index 00000000000..2c76d5ec547
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/RawTestCase.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.*;
+
+/**
+ * Created by balder on 10/21/14.
+ */
+public class RawTestCase {
+    @Test
+    public void requireThatAssignWorks() {
+        byte [] buf = {0,1,2,3};
+        byte [] empty = {};
+        ByteBuffer bb = ByteBuffer.wrap(buf);
+        Raw a = new Raw();
+        a.assign(buf);
+        assertEquals(bb, a.getWrappedValue());
+        a.assign(empty);
+        assertNotEquals(bb, a.getWrappedValue());
+        a.assign(bb);
+        assertEquals(bb, a.getWrappedValue());
+        a.assign(empty);
+        assertNotEquals(bb, a.getWrappedValue());
+        a.assign(new Raw(bb));
+        assertEquals(bb, a.getWrappedValue());
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/StringFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/StringFieldValueTestCase.java
new file mode 100644
index 00000000000..8a0bfe68e72
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/StringFieldValueTestCase.java
@@ -0,0 +1,393 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import org.junit.Test;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.14
+ */
+public class StringFieldValueTestCase {
+
+    @Test
+    public void requireThatCharWorks() {
+        new StringFieldValue("\t");
+        new StringFieldValue("\r");
+        new StringFieldValue("\n");
+        for (int c = 0x20; c < 0xFDD0; c++) {
+            new StringFieldValue("" + Character.toChars(c)[0]);
+        }
+        for (int c = 0xFDE0; c < 0xFFFF; c++) {
+            new StringFieldValue("" + Character.toChars(c)[0]);
+        }
+        for (int c = 0x10000; c < 0x1FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x20000; c < 0x2FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x30000; c < 0x3FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x40000; c < 0x4FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x50000; c < 0x5FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x60000; c < 0x6FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x70000; c < 0x7FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x80000; c < 0x8FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x90000; c < 0x9FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0xA0000; c < 0xAFFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0xB0000; c < 0xBFFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0xC0000; c < 0xCFFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0xD0000; c < 0xDFFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0xE0000; c < 0xEFFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0xF0000; c < 0xFFFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+        for (int c = 0x100000; c < 0x10FFFE; c++) {
+            char[] chars = Character.toChars(c);
+            new StringFieldValue("" + chars[0] + chars[1]);
+        }
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails0() {
+        new StringFieldValue("\u0000");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1() {
+        new StringFieldValue("\u0001");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails2() {
+        new StringFieldValue("\u0002");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails3() {
+        new StringFieldValue("\u0003");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails4() {
+        new StringFieldValue("\u0004");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails5() {
+        new StringFieldValue("\u0005");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails6() {
+        new StringFieldValue("\u0006");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails7() {
+        new StringFieldValue("\u0007");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsB() {
+        new StringFieldValue("\u000B");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsC() {
+        new StringFieldValue("\u000C");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsE() {
+        new StringFieldValue("\u000E");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsF() {
+        new StringFieldValue("\u000F");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails10() {
+        new StringFieldValue("\u0010");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails11() {
+        new StringFieldValue("\u0011");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails12() {
+        new StringFieldValue("\u0012");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails13() {
+        new StringFieldValue("\u0013");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails14() {
+        new StringFieldValue("\u0014");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails15() {
+        new StringFieldValue("\u0015");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails16() {
+        new StringFieldValue("\u0016");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails17() {
+        new StringFieldValue("\u0017");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails18() {
+        new StringFieldValue("\u0018");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails19() {
+        new StringFieldValue("\u0019");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1A() {
+        new StringFieldValue("\u001A");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1B() {
+        new StringFieldValue("\u001B");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1C() {
+        new StringFieldValue("\u001C");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1D() {
+        new StringFieldValue("\u001D");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1E() {
+        new StringFieldValue("\u001E");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1F() {
+        new StringFieldValue("\u001F");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD0() {
+        new StringFieldValue("\uFDD0");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD1() {
+        new StringFieldValue("\uFDD1");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD2() {
+        new StringFieldValue("\uFDD2");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD3() {
+        new StringFieldValue("\uFDD3");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD4() {
+        new StringFieldValue("\uFDD4");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD5() {
+        new StringFieldValue("\uFDD5");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD6() {
+        new StringFieldValue("\uFDD6");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD7() {
+        new StringFieldValue("\uFDD7");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD8() {
+        new StringFieldValue("\uFDD8");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDD9() {
+        new StringFieldValue("\uFDD9");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDDA() {
+        new StringFieldValue("\uFDDA");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDDB() {
+        new StringFieldValue("\uFDDB");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDDC() {
+        new StringFieldValue("\uFDDC");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDDD() {
+        new StringFieldValue("\uFDDD");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDDE() {
+        new StringFieldValue("\uFDDE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFDDF() {
+        new StringFieldValue("\uFDDF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1FFFE() {
+        new StringFieldValue("\uD83F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails1FFFF() {
+        new StringFieldValue("\uD83F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails2FFFE() {
+        new StringFieldValue("\uD87F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails2FFFF() {
+        new StringFieldValue("\uD87F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails3FFFE() {
+        new StringFieldValue("\uD8BF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails3FFFF() {
+        new StringFieldValue("\uD8BF\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails4FFFE() {
+        new StringFieldValue("\uD8FF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails4FFFF() {
+        new StringFieldValue("\uD8FF\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails5FFFE() {
+        new StringFieldValue("\uD93F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails5FFFF() {
+        new StringFieldValue("\uD93F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails6FFFE() {
+        new StringFieldValue("\uD97F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails6FFFF() {
+        new StringFieldValue("\uD97F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails7FFFE() {
+        new StringFieldValue("\uD9BF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails7FFFF() {
+        new StringFieldValue("\uD9BF\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails8FFFE() {
+        new StringFieldValue("\uD9FF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails8FFFF() {
+        new StringFieldValue("\uD9FF\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails9FFFE() {
+        new StringFieldValue("\uDA3F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails9FFFF() {
+        new StringFieldValue("\uDA3F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsAFFFE() {
+        new StringFieldValue("\uDA7F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsAFFFF() {
+        new StringFieldValue("\uDA7F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsBFFFE() {
+        new StringFieldValue("\uDABF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsBFFFF() {
+        new StringFieldValue("\uDABF\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsCFFFE() {
+        new StringFieldValue("\uDAFF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsCFFFF() {
+        new StringFieldValue("\uDAFF\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsDFFFE() {
+        new StringFieldValue("\uDB3F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsDFFFF() {
+        new StringFieldValue("\uDB3F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsEFFFE() {
+        new StringFieldValue("\uDB7F\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsEFFFF() {
+        new StringFieldValue("\uDB7F\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFFFFE() {
+        new StringFieldValue("\uDBBF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFailsFFFFF() {
+        new StringFieldValue("\uDBBF\uDFFF");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails10FFFE() {
+        new StringFieldValue("\uDBFF\uDFFE");
+    }
+    @Test(expected = IllegalArgumentException.class)
+    public void requireThatControlCharFails10FFFF() {
+        new StringFieldValue("\uDBFF\uDFFF");
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java
new file mode 100644
index 00000000000..cca39ee5bc4
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java
@@ -0,0 +1,434 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.Field;
+import com.yahoo.document.annotation.AbstractTypesTest;
+import com.yahoo.document.annotation.Annotation;
+import com.yahoo.document.annotation.AnnotationType;
+import com.yahoo.document.annotation.AnnotationTypeRegistry;
+import com.yahoo.document.annotation.Span;
+import com.yahoo.document.annotation.SpanList;
+import com.yahoo.document.annotation.SpanNode;
+import com.yahoo.document.annotation.SpanTree;
+import com.yahoo.document.serialization.*;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.vespa.objects.BufferSerializer;
+import org.junit.Test;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.ListIterator;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class StringTestCase extends AbstractTypesTest {
+
+    @Test
+    public void testDeserialize() throws Exception {
+        byte[] buf = new byte[1500];
+        GrowableByteBuffer data = GrowableByteBuffer.wrap(buf);
+        //short string
+        java.lang.String foo = "foo";
+
+        data.put((byte)0);
+        data.put((byte)(foo.length() + 1));
+
+        data.put(foo.getBytes());
+        data.put((byte)0);
+
+        int positionAfterPut = data.position();
+
+        data.position(0);
+
+        StringFieldValue tmp = new StringFieldValue();
+        DocumentDeserializer deser = DocumentDeserializerFactory.create42(null, data);
+        tmp.deserialize(deser);
+        java.lang.String foo2 = tmp.getString();
+
+        assertTrue(foo.equals(foo2));
+        assertEquals(data.position(), positionAfterPut);
+
+        //=====================
+
+        buf = new byte[1500];
+        data = GrowableByteBuffer.wrap(buf);
+
+        //long string
+        java.lang.String blah = "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah" +
+                                "blahblahblahblahblahblahblahblahblahblahblah";
+
+        int length = blah.length() + 1;
+
+        data.put((byte)0);
+        data.putInt(length | 0x80000000);
+
+        data.put(blah.getBytes());
+        data.put((byte)0);
+
+        positionAfterPut = data.position();
+
+        data.position(0);
+
+        tmp = new StringFieldValue();
+
+        deser = DocumentDeserializerFactory.create42(null, data);
+        tmp.deserialize(deser);
+        java.lang.String blah2 = tmp.getString();
+
+        assertEquals(data.position(), positionAfterPut);
+        assertTrue(blah.equals(blah2));
+    }
+
+/*    public void testSpanTree() {
+        StringFieldValue annotatedText = new StringFieldValue("banana airlines");
+        SpanList lingTree = new SpanList();
+        annotatedText.setSpanTree("linguistics", lingTree);
+        for (Annotation anAnnotation : annotatedText.getSpanTree("linguistics")) {
+            System.err.println(anAnnotation);
+        }
+    }*/
+
+    @Test
+    public void testSerializeDeserialize() throws Exception {
+        java.lang.String test = "Hello hello";
+        BufferSerializer data = new BufferSerializer(new GrowableByteBuffer(100, 2.0f));
+        StringFieldValue value = new StringFieldValue(test);
+        value.serialize(data);
+
+        data.getBuf().position(0);
+
+        StringFieldValue tmp = new StringFieldValue();
+        DocumentDeserializer deser = DocumentDeserializerFactory.create42(null, data.getBuf());
+        tmp.deserialize(deser);
+        java.lang.String test2 = tmp.getString();
+        assertEquals(test, test2);
+    }
+
+    @Test
+    public void testSerializationWithTree() {
+        StringFieldValue text = getAnnotatedString();
+
+        serializeAndAssert(text);
+    }
+
+    private void serializeAndAssert(StringFieldValue stringFieldValue) {
+        Field f = new Field("text", DataType.STRING);
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+        serializer.write(f, stringFieldValue);
+        buffer.flip();
+
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
+        StringFieldValue stringFieldValue2 = new StringFieldValue();
+        deserializer.read(f, stringFieldValue2);
+
+        assertEquals(stringFieldValue, stringFieldValue2);
+        assertNotSame(stringFieldValue, stringFieldValue2);
+    }
+
+    @Test
+    public void testNestedSpanTreeBug4187377() {
+        AnnotationType type = new AnnotationType("ann", DataType.STRING);
+
+        StringFieldValue outerString = new StringFieldValue("Ballooo");
+        SpanTree outerTree = new SpanTree("outer");
+        outerString.setSpanTree(outerTree);
+
+        SpanList outerRoot = (SpanList)outerTree.getRoot();
+        Span outerSpan = new Span(0, 1);
+        outerRoot.add(outerSpan);
+
+        StringFieldValue innerString = new StringFieldValue("innerBalloooo");
+
+        outerTree.annotate(outerSpan, new Annotation(type, innerString));
+
+        SpanTree innerTree = new SpanTree("inner");
+        innerString.setSpanTree(innerTree);
+
+        SpanList innerRoot = (SpanList)innerTree.getRoot();
+        Span innerSpan = new Span(0, 1);
+        innerRoot.add(innerSpan);
+        innerTree.annotate(innerSpan, new Annotation(type));
+
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+
+        try {
+            serializer.write(null, outerString);
+            fail("Should have failed, nested span trees are not supported.");
+        } catch (SerializationException se) {
+            //OK!
+        }
+    }
+
+    /**
+     * Test for bug 4066566. No assertions, but works if it runs without exceptions.
+     */
+    @Test
+    public void testAnnotatorConsumer() {
+        DocumentTypeManager manager = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer
+                .configure(manager, "file:src/test/java/com/yahoo/document/datatypes/documentmanager.blog.sd");
+
+        DocumentType blogType = manager.getDocumentType("blog");
+        Document doc = new Document(blogType, "doc:blog:http://blogs.sun.com/praveenm");
+        doc.setFieldValue("url", new StringFieldValue("http://blogs.sun.com/praveenm"));
+        doc.setFieldValue("title", new StringFieldValue("Beginning JavaFX"));
+        doc.setFieldValue("author", new StringFieldValue("Praveen Mohan"));
+        doc.setFieldValue("body", new StringFieldValue(
+                "JavaFX can expand its wings across different domains such as manufacturing, logistics, retail, etc. Many companies have adopted it - IBM, Oracle, Yahoo, Honeywell. Even the non-IT industries such as GE, WIPRO, Ford etc. So it is a success for Christopher Oliver and Richard Bair. Scott Mcnealy is happy"));
+
+        doc = annotate(doc, manager);
+        doc = serializeAndDeserialize(doc, manager);
+        doc = consume(doc, manager);
+        System.err.println(doc);
+    }
+
+    private Document serializeAndDeserialize(Document doc, DocumentTypeManager manager) {
+        GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
+        DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
+        serializer.write(doc);
+        buffer.flip();
+
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer);
+        return new Document(deserializer);
+    }
+
+    public Document annotate(Document document, DocumentTypeManager manager) {
+        AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();
+
+        AnnotationType company = registry.getType("company");
+        AnnotationType industry = registry.getType("industry");
+        AnnotationType person = registry.getType("person");
+        AnnotationType location = registry.getType("location");
+
+        Map<String, AnnotationType> m = registry.getTypes();
+        for (String key : m.keySet()) {
+            System.out.println("Key: " + key);
+            AnnotationType val = m.get(key);
+            parseAnnotationType(val);
+        }
+
+        SpanTree tree = new SpanTree("testannotations");
+        SpanList root = (SpanList)tree.getRoot();
+
+        SpanNode companySpan = new Span(0, 5);
+        SpanNode industrySpan = new Span(5, 10);
+        SpanNode personSpan = new Span(10, 15);
+        SpanNode locationSpan = new Span(15, 20);
+
+        root.add(companySpan);
+        root.add(industrySpan);
+        root.add(personSpan);
+        root.add(locationSpan);
+
+        Struct companyValue = (Struct)company.getDataType().createFieldValue();
+        companyValue.setFieldValue("name", new StringFieldValue("Sun"));
+        companyValue.setFieldValue("ceo", new StringFieldValue("Scott Mcnealy"));
+        companyValue.setFieldValue("lat", new DoubleFieldValue(37.7));
+        companyValue.setFieldValue("lon", new DoubleFieldValue(-122.44));
+        companyValue.setFieldValue("vertical", new StringFieldValue("software"));
+        Annotation compAn = new Annotation(company, companyValue);
+        tree.annotate(companySpan, compAn);
+
+        Struct personValue = new Struct(manager.getDataType("annotation.person"));
+        personValue.setFieldValue("name", new StringFieldValue("Richard Bair"));
+        Annotation personAn = new Annotation(person, personValue);
+        tree.annotate(personSpan, personAn);
+
+        Struct locValue = new Struct(manager.getDataType("annotation.location"));
+        locValue.setFieldValue("name", new StringFieldValue("Prinsens Gate"));
+        Annotation loc = new Annotation(location, locValue);
+        tree.annotate(locationSpan, loc);
+
+        Struct locValue2 = new Struct(manager.getDataType("annotation.location"));
+        locValue2.setFieldValue("name", new StringFieldValue("Kongens Gate"));
+        Annotation locAn = new Annotation(location, locValue2);
+        tree.annotate(locationSpan, locAn);
+
+        SpanList branch = new SpanList();
+
+        SpanNode span1 = new Span(0, 3);
+        SpanNode span2 = new Span(1, 9);
+        SpanNode span3 = new Span(12, 10);
+
+        branch.add(span1);
+        branch.add(span3);
+        branch.add(span2);
+
+        Struct industryValue = new Struct(manager.getDataType("annotation.industry"));
+        industryValue.setFieldValue("vertical", new StringFieldValue("Manufacturing"));
+        Annotation ind = new Annotation(industry, industryValue);
+        tree.annotate(span1, ind);
+
+        Struct pValue = new Struct(manager.getDataType("annotation.person"));
+        pValue.setFieldValue("name", new StringFieldValue("Praveen Mohan"));
+        Annotation pAn = new Annotation(person, pValue);
+        tree.annotate(span2, pAn);
+
+        Struct lValue = new Struct(manager.getDataType("annotation.location"));
+        lValue.setFieldValue("name", new StringFieldValue("Embassy Golf Links"));
+        Annotation locn = new Annotation(location, lValue);
+        tree.annotate(span3, locn);
+
+        Struct cValue = (Struct)company.getDataType().createFieldValue();
+        cValue.setFieldValue("name", new StringFieldValue("Yahoo"));
+        cValue.setFieldValue("ceo", new StringFieldValue("Carol Bartz"));
+        cValue.setFieldValue("lat", new DoubleFieldValue(127.7));
+        cValue.setFieldValue("lon", new DoubleFieldValue(-42.44));
+        cValue.setFieldValue("vertical", new StringFieldValue("search"));
+        Annotation cAn = new Annotation(company, cValue);
+        tree.annotate(branch, cAn);
+
+        Struct pVal = new Struct(manager.getDataType("annotation.person"));
+        pVal.setFieldValue("name", new StringFieldValue("Kim Omar"));
+        Annotation an = new Annotation(person, pVal);
+        tree.annotate(root, an);
+        root.add(branch);
+
+        StringFieldValue body = (StringFieldValue)document.getFieldValue(document.getDataType().getField("body"));
+
+        root.remove(branch);
+        tree.cleanup();
+
+        System.out.println("No. Of Annotations: " + tree.numAnnotations());
+        body.setSpanTree(tree);
+
+        document.setFieldValue(document.getField("body"), body);
+
+        return document;
+    }
+
+    public Document consume(Document document, DocumentTypeManager docTypeMgr) {
+        DocumentType type = docTypeMgr.getDocumentType("blog");
+        Collection<Field> fc = type.getFields();
+        for (Field f : fc) {
+            System.out.println("\n\nField Name: " + f.getName());
+            System.out.println("DataType: " + f.getDataType());
+            System.out.println("isHeader? " + f.isHeader());
+
+            FieldValue val = document.getFieldValue(f);
+            if (val instanceof StringFieldValue) {
+                StringFieldValue sfv = (StringFieldValue)val;
+                System.out.println(f.getName() + " is a StringField. Field Value: " + sfv.getString());
+                Collection<SpanTree> c = sfv.getSpanTrees();
+                for (SpanTree tree : c) {
+                    System.out.println(f.getName() + " has annotations");
+                    consumeAnnotations(tree, (SpanList)tree.getRoot());
+                }
+            }
+        }
+
+        return document;
+    }
+
+    public void consumeAnnotations(SpanTree tree, SpanList root) {
+        System.out.println("\n\nSpanList: " + root + " num Children: " + root.numChildren());
+        System.out.println("-------------------");
+        Iterator<SpanNode> childIterator = root.childIterator();
+        while (childIterator.hasNext()) {
+            SpanNode node = childIterator.next();
+            System.out.println("Span Node: " + node); // + " Span Text: " + node.getText(fieldValStr));
+            if (node instanceof SpanList) {
+                System.out.println("Encountered another span list");
+                SpanList spl = (SpanList)node;
+                ListIterator<SpanNode> lli = spl.childIterator();
+                while (lli.hasNext()) {
+                    System.out.print(" " + lli.next() + " ");
+                }
+                consumeAnnotations(tree, (SpanList)node);
+            } else {
+                System.out.println("\nGetting annotations for this span node: " + node);
+                getAnnotationsForNode(tree, node);
+            }
+        }
+        System.out.println("\nGetting annotations for the SpanList itself : " + root);
+        getAnnotationsForNode(tree, root);
+    }
+
+    public void getAnnotationsForNode(SpanTree tree, SpanNode node) {
+        Iterator<Annotation> iter = tree.iterator(node);
+        boolean annotationPresent = false;
+        while (iter.hasNext()) {
+            annotationPresent = true;
+            Annotation xx = iter.next();
+            Struct fValue = (Struct)xx.getFieldValue();
+            System.out.println("Annotation: " + xx);
+            if (fValue == null) {
+                System.out.println("Field Value is null");
+                return;
+            }
+            Iterator fieldIter = fValue.iterator();
+            while (fieldIter.hasNext()) {
+                Map.Entry m = (Map.Entry)fieldIter.next();
+                Field f = (Field)m.getKey();
+                FieldValue val = (FieldValue)m.getValue();
+                System.out.println("Field : " + f + " Value: " + val);
+            }
+        }
+        if (!annotationPresent) {
+            System.out.println("****No annotations found for the span node: " + node);
+        }
+    }
+
+    public void parseAnnotationType(AnnotationType t) {
+        System.out.println("Type Name: " + t.getName());
+        System.out.println("Type ID: " + t.getId());
+        DataType dt = t.getDataType();
+        String dataTypeStr;
+        if (dt == DataType.STRING) {
+            dataTypeStr = "String";
+        } else if (dt == DataType.INT) {
+            dataTypeStr = "Integer";
+        } else if (dt == DataType.URI) {
+            dataTypeStr = "URL";
+        } else {
+            dataTypeStr = "UNKNOWN";
+        }
+        System.out.println("Type DataType: " + dataTypeStr);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java
new file mode 100644
index 00000000000..76e1588b2a1
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java
@@ -0,0 +1,356 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class StructTestCase {
+    @Test
+    public void testBasicStuff() throws Exception {
+        StructDataType type = new StructDataType("teststr");
+        type.addField(new Field("int", 0, DataType.INT, true));
+        type.addField(new Field("flt", 1, DataType.FLOAT, true));
+        type.addField(new Field("str", 2, DataType.STRING, true));
+        type.addField(new Field("raw", 3, DataType.RAW, true));
+        type.addField(new Field("lng", 4, DataType.LONG, true));
+        type.addField(new Field("dbl", 5, DataType.DOUBLE, true));
+        type.addField(new Field("uri", 6, DataType.URI, true));
+        type.addField(new Field("byt", 8, DataType.BYTE, true));
+
+        Struct struct = new Struct(type);
+        {
+            //add and remove again:
+            assertEquals(0, struct.getFields().size());
+            IntegerFieldValue ifv = new IntegerFieldValue(5);
+            struct.setFieldValue("int", ifv);
+            assertEquals(1, struct.getFields().size());
+            struct.removeFieldValue("int");
+            assertEquals(0, struct.getFields().size());
+        }
+
+        {
+            //add three elements and remove one of them, and replace the last one:
+            assertEquals(0, struct.getFields().size());
+
+            IntegerFieldValue ifv = new IntegerFieldValue(5);
+            struct.setFieldValue("int", ifv);
+            assertEquals(1, struct.getFields().size());
+
+            FloatFieldValue ffv = new FloatFieldValue(5.0f);
+            struct.setFieldValue("flt", ffv);
+            assertEquals(2, struct.getFields().size());
+
+            DoubleFieldValue dfv = new DoubleFieldValue(6.0d);
+            struct.setFieldValue("dbl", dfv);
+            assertEquals(3, struct.getFields().size());
+
+            Iterator<Map.Entry<Field, FieldValue>> it = struct.iterator();
+            assertSame(ifv, it.next().getValue());
+            assertSame(ffv, it.next().getValue());
+            assertSame(dfv, it.next().getValue());
+            assertFalse(it.hasNext());
+
+            struct.removeFieldValue("flt");
+            assertEquals(2, struct.getFields().size());
+
+            it = struct.iterator();
+            assertSame(ifv, it.next().getValue());
+            assertSame(dfv, it.next().getValue());
+            assertFalse(it.hasNext());
+
+            DoubleFieldValue dfv2 = new DoubleFieldValue(9.0d);
+            struct.setFieldValue("dbl", dfv2);
+            assertEquals(2, struct.getFields().size());
+
+            it = struct.iterator();
+            assertSame(ifv, it.next().getValue());
+            assertSame(dfv2, it.next().getValue());
+            assertFalse(it.hasNext());
+        }
+    }
+
+    @Test
+    public void testSetGetPrimitiveTypes() throws Exception {
+        StructDataType type = new StructDataType("teststr");
+        type.addField(new Field("int", DataType.INT));
+        type.addField(new Field("flt", DataType.FLOAT));
+        type.addField(new Field("str", DataType.STRING));
+        type.addField(new Field("raw", DataType.RAW));
+        type.addField(new Field("lng", DataType.LONG));
+        type.addField(new Field("dbl", DataType.DOUBLE));
+        type.addField(new Field("uri", DataType.URI));
+        type.addField(new Field("byt", DataType.BYTE));
+
+        Struct struct = new Struct(type);
+
+        {
+            IntegerFieldValue nt = new IntegerFieldValue(544);
+            Object o = struct.setFieldValue("int", nt);
+            assertNull(o);
+            assertEquals(new IntegerFieldValue(544), struct.getFieldValue("int"));
+            o = struct.setFieldValue("int", 500);
+            assertEquals(nt, o);
+            assertFalse(nt.equals(struct.getFieldValue("int")));
+        }
+        {
+            FloatFieldValue flt = new FloatFieldValue(5.44f);
+            Object o = struct.setFieldValue("flt", flt);
+            assertNull(o);
+            assertEquals(flt, struct.getFieldValue("flt"));
+            o = struct.setFieldValue("flt", new FloatFieldValue(5.00f));
+            assertEquals(flt, o);
+            assertFalse(flt.equals(struct.getFieldValue("flt")));
+        }
+        {
+            StringFieldValue string = new StringFieldValue("this is a string");
+            Object o = struct.setFieldValue("str", string);
+            assertNull(o);
+            assertEquals(string, struct.getFieldValue("str"));
+            o = struct.setFieldValue("str", "another string");
+            assertEquals(string, o);
+            assertSame(string, o);
+            assertFalse(string.equals(struct.getFieldValue("str")));
+        }
+        {
+            Raw buf = new Raw(ByteBuffer.wrap(new byte[100]));
+            Object o = struct.setFieldValue("raw", buf);
+            assertNull(o);
+            assertEquals(buf, struct.getFieldValue("raw"));
+            o = struct.setFieldValue("raw", new Raw(ByteBuffer.wrap(new byte[50])));
+            assertEquals(buf, o);
+            assertSame(buf, o);
+            assertFalse(buf.equals(struct.getFieldValue("raw")));
+        }
+        {
+            LongFieldValue lng = new LongFieldValue(59879879879079L);
+            Object o = struct.setFieldValue("lng", lng);
+            assertEquals(lng, struct.getFieldValue("lng"));
+            o = struct.setFieldValue("lng", new LongFieldValue(23418798734243L));
+            assertEquals(lng, o);
+            assertFalse(lng.equals(struct.getFieldValue("lng")));
+        }
+        {
+            DoubleFieldValue dbl = new DoubleFieldValue(5.44d);
+            Object o = struct.setFieldValue("dbl", dbl);
+            assertNull(o);
+            assertEquals(dbl, struct.getFieldValue("dbl"));
+            o = struct.setFieldValue("dbl", new DoubleFieldValue(5.00d));
+            assertEquals(dbl, o);
+            assertFalse(dbl.equals(struct.getFieldValue("dbl")));
+        }
+        {
+            UriFieldValue uri = new UriFieldValue("this is a uri");
+            Object o = struct.setFieldValue("uri", uri);
+            assertNull(o);
+            assertEquals(uri, struct.getFieldValue("uri"));
+            o = struct.setFieldValue("uri", "another uri");
+            assertEquals(uri, o);
+            assertSame(uri, o);
+            assertFalse(uri.equals(struct.getFieldValue("uri")));
+        }
+        {
+            ByteFieldValue byt = new ByteFieldValue((byte)123);
+            Object o = struct.setFieldValue("byt", byt);
+            assertNull(o);
+            assertEquals(byt, struct.getFieldValue("byt"));
+            o = struct.setFieldValue("byt", (byte) 100);
+            assertEquals(byt, o);
+            assertFalse(byt.equals(struct.getFieldValue("byt")));
+        }
+    }
+
+    @Test
+    public void testSetGetAggregateTypes() throws Exception {
+        StructDataType type = new StructDataType("teststr");
+        type.addField(new Field("intarray", DataType.getArray(DataType.INT)));
+        type.addField(new Field("strws", DataType.getWeightedSet(DataType.STRING)));
+
+        Struct struct = new Struct(type);
+
+        {
+            //TEST USING OUR IMPLEMENTATION OF LIST
+            Array integerArray = new Array(type.getField("intarray").getDataType());
+            integerArray.add(new IntegerFieldValue(5));
+            integerArray.add(new IntegerFieldValue(10));
+            assertEquals(2, integerArray.size());
+            struct.setFieldValue("intarray", integerArray);
+            assertEquals(2, integerArray.size());
+            List outList = (List) struct.getFieldValue("intarray");
+            integerArray.add(new IntegerFieldValue(322));
+            integerArray.add(new IntegerFieldValue(453));
+            assertEquals(integerArray, outList);
+            assertSame(integerArray, outList);
+            assertEquals(4, integerArray.size());
+            Array anotherArray = new Array(type.getField("intarray").getDataType());
+            anotherArray.add(new IntegerFieldValue(5324));
+            Object o = struct.setFieldValue("intarray", anotherArray);
+            assertEquals(integerArray, o);
+            assertSame(integerArray, o);
+            outList = (List) struct.getFieldValue("intarray");
+            assertFalse(integerArray.equals(outList));
+            assertEquals(anotherArray, outList);
+            assertSame(anotherArray, outList);
+        }
+        {
+            WeightedSet<StringFieldValue> strWs = new WeightedSet<>(type.getField("strws").getDataType());
+            strWs.put(new StringFieldValue("banana"), 10);
+            strWs.add(new StringFieldValue("apple"));
+            assertEquals(2, strWs.size());
+            Object o = struct.setFieldValue("strws", strWs);
+            assertNull(o);
+            assertEquals(2, strWs.size());
+            WeightedSet<StringFieldValue> outWs = (WeightedSet<StringFieldValue>) struct.getFieldValue("strws");
+            strWs.add(new StringFieldValue("poison"));
+            strWs.put(new StringFieldValue("pie"), 599);
+            assertEquals(strWs, outWs);
+            assertSame(strWs, outWs);
+            assertEquals(4, strWs.size());
+            WeightedSet anotherWs = new WeightedSet(type.getField("strws").getDataType());
+            anotherWs.add(new StringFieldValue("be bop"));
+            o = struct.setFieldValue("strws", anotherWs);
+            assertEquals(strWs, o);
+            assertSame(strWs, o);
+            outWs = (WeightedSet<StringFieldValue>) struct.getFieldValue("strws");
+
+            System.out.println("OutWS " + outWs);
+            System.out.println("StrWS " + strWs);
+
+            assertFalse(strWs.equals(outWs));
+            assertEquals(anotherWs, outWs);
+            assertSame(anotherWs, outWs);
+        }
+    }
+
+    @Test
+    public void testSetUnknownType() {
+        StructDataType type = new StructDataType("teststr");
+        type.addField(new Field("int", 0, DataType.INT, true));
+
+        Struct struct = new Struct(type);
+        try {
+            struct.setFieldValue(new Field("alien", DataType.STRING), new StringFieldValue("foo"));
+            fail("Alien type worked");
+        } catch (IllegalArgumentException expected) {
+            assertTrue(expected.getMessage().matches(".*No such field.*"));
+        }
+    }
+
+    @Test
+    public void testCompareToDoesNotMutateStateBug6394548() {
+        StructDataType type = new StructDataType("test");
+        // NOTE: non-increasing ID order!
+        type.addField(new Field("int", 2, DataType.INT, true));
+        type.addField(new Field("flt", 1, DataType.FLOAT, true));
+        type.addField(new Field("str", 0, DataType.STRING, true));
+
+        Struct a = new Struct(type);
+        a.setFieldValue("int", new IntegerFieldValue(123));
+        a.setFieldValue("flt", new DoubleFieldValue(45.6));
+        a.setFieldValue("str", new StringFieldValue("hello world"));
+        Struct b = new Struct(type);
+        b.setFieldValue("int", new IntegerFieldValue(100));
+        b.setFieldValue("flt", new DoubleFieldValue(45.6));
+        b.setFieldValue("str", new StringFieldValue("hello world"));
+
+        String xmlBefore = a.toXml();
+        int hashBefore = a.hashCode();
+
+        assertEquals(1, a.compareTo(b));
+
+        assertEquals(xmlBefore, a.toXml());
+        assertEquals(hashBefore, a.hashCode());
+    }
+
+    @Test
+    public void sortingFirstOrderedByNumberOfSetFields() {
+        StructDataType type = new StructDataType("test");
+        type.addField(new Field("int", DataType.INT));
+        type.addField(new Field("flt", DataType.FLOAT));
+        type.addField(new Field("str", DataType.STRING));
+
+        Struct a = new Struct(type);
+        a.setFieldValue("int", new IntegerFieldValue(123));
+        Struct b = new Struct(type);
+
+        assertTrue(b.compareTo(a) < 0);
+        assertTrue(a.compareTo(b) > 0);
+        assertEquals(0, b.compareTo(b));
+        assertFalse(a.equals(b));
+        assertFalse(b.equals(a));
+
+        b.setFieldValue("int", new IntegerFieldValue(123));
+
+        assertEquals(0, a.compareTo(b));
+        assertEquals(0, b.compareTo(a));
+        assertTrue(a.equals(b));
+        assertTrue(b.equals(a));
+
+        b.setFieldValue("str", new StringFieldValue("hello world"));
+        assertTrue(b.compareTo(a) > 0);
+        assertTrue(a.compareTo(b) < 0);
+        assertFalse(a.equals(b));
+        assertFalse(b.equals(a));
+    }
+
+    @Test
+    public void sortingOrderIndependentOfValueInsertionOrder() {
+        StructDataType type = new StructDataType("test");
+        type.addField(new Field("int", DataType.INT));
+        type.addField(new Field("flt", DataType.FLOAT));
+        type.addField(new Field("str", DataType.STRING));
+
+        Struct a = new Struct(type);
+        a.setFieldValue("int", new IntegerFieldValue(123));
+        a.setFieldValue("flt", new DoubleFieldValue(45.6));
+        a.setFieldValue("str", new StringFieldValue("hello world"));
+        Struct b = new Struct(type);
+        b.setFieldValue("str", new StringFieldValue("hello world"));
+        b.setFieldValue("flt", new DoubleFieldValue(45.6));
+        b.setFieldValue("int", new IntegerFieldValue(123));
+
+        assertEquals(0, a.compareTo(b));
+        assertEquals(0, b.compareTo(a));
+        assertTrue(a.equals(b));
+        assertTrue(b.equals(a));
+
+        b.setFieldValue("int", new IntegerFieldValue(122));
+        assertTrue(a.compareTo(b) > 0);
+        assertTrue(b.compareTo(a) < 0);
+        assertFalse(a.equals(b));
+        assertFalse(b.equals(a));
+    }
+
+    @Test
+    public void sortingOrderDependsOnTypeFieldOrderWhenNonEqual() {
+        StructDataType type = new StructDataType("test");
+        type.addField(new Field("int", DataType.INT));
+        type.addField(new Field("intnotset", DataType.INT));
+        type.addField(new Field("flt", DataType.FLOAT));
+        type.addField(new Field("str", DataType.STRING));
+
+        Struct a = new Struct(type);
+        a.setFieldValue("int", new IntegerFieldValue(123));
+        a.setFieldValue("flt", new DoubleFieldValue(45.6));
+        Struct b = new Struct(type);
+        b.setFieldValue("int", new IntegerFieldValue(123));
+        b.setFieldValue("str", new StringFieldValue("hello world"));
+
+        // a sorts before b as it has flt set which occurs before str in the type
+        assertTrue(a.compareTo(b) < 0);
+        assertTrue(b.compareTo(a) > 0);
+        assertFalse(a.equals(b));
+        assertFalse(b.equals(a));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java
new file mode 100644
index 00000000000..7c01f8ab300
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/TensorFieldValueTestCase.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.tensor.MapTensor;
+import com.yahoo.tensor.Tensor;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class TensorFieldValueTestCase {
+
+    private static TensorFieldValue createFieldValue(String tensor) {
+        return new TensorFieldValue(MapTensor.from(tensor));
+    }
+
+    @Test
+    public void requireThatDifferentFieldValueTypesAreNotEqual() {
+        assertFalse(createFieldValue("{{x:0}:2.0}").equals(new IntegerFieldValue(5)));
+    }
+
+    @Test
+    public void requireThatDifferentTensorValuesAreNotEqual() {
+        TensorFieldValue lhs = createFieldValue("{{x:0}:2.0}");
+        TensorFieldValue rhs = createFieldValue("{{x:0}:3.0}");
+        assertFalse(lhs.equals(rhs));
+        assertFalse(lhs.equals(new TensorFieldValue()));
+    }
+
+    @Test
+    public void requireThatSameTensorValueIsEqual() {
+        Tensor tensor = MapTensor.from("{{x:0}:2.0}");
+        TensorFieldValue lhs = new TensorFieldValue(tensor);
+        TensorFieldValue rhs = new TensorFieldValue(tensor);
+        assertTrue(lhs.equals(lhs));
+        assertTrue(lhs.equals(rhs));
+        assertTrue(lhs.equals(createFieldValue("{{x:0}:2.0}")));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/UriFieldValueTest.java b/document/src/test/java/com/yahoo/document/datatypes/UriFieldValueTest.java
new file mode 100644
index 00000000000..8fae87e5a29
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/UriFieldValueTest.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import org.junit.Test;
+
+import java.net.URI;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class UriFieldValueTest {
+
+    @Test
+    public void requireThatURICanBeAssigned() {
+        UriFieldValue value = new UriFieldValue();
+        String uri = "http://user:pass@localhost:69/path#fragment?query";
+        value.assign(URI.create(uri));
+        assertEquals(uri, value.getWrappedValue());
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/WeightedSetTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/WeightedSetTestCase.java
new file mode 100644
index 00000000000..fb2122d9828
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/WeightedSetTestCase.java
@@ -0,0 +1,160 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.datatypes;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.MapDataType;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class WeightedSetTestCase extends junit.framework.TestCase {
+    public void testSet() {
+        WeightedSet<StringFieldValue> wset = new WeightedSet<>(DataType.TAG);
+
+        //ADD:
+
+        Object ok;
+        ok = wset.put(new StringFieldValue("this is a test"), 5);
+        assertNull(ok);
+        ok = wset.put(new StringFieldValue("this is a test"), 10);
+        assertEquals(5, ok);
+
+        assertEquals(1, wset.size());
+        assertEquals(wset.get(new StringFieldValue("this is a test")), new Integer(10));
+
+        //REMOVE:
+
+        ok = wset.put(new StringFieldValue("another test"), 7);
+        assertNull(ok);
+
+        assertEquals(2, wset.size());
+
+        ok = wset.remove(new StringFieldValue("this is a test"));
+        assertNotNull(ok);
+        assertEquals(1, wset.size());
+
+        ok = wset.remove(new StringFieldValue("another test"));
+        assertNotNull(ok);
+        assertEquals(0, wset.size());
+
+        //CONTAINS:
+
+        wset.put(new StringFieldValue("ballooo"), 50);
+        wset.put(new StringFieldValue("bananaa"), 51);
+
+        ok = wset.containsKey(new StringFieldValue("bananaa"));
+        assertEquals(true, ok);
+        ok = wset.containsKey(new StringFieldValue("ballooo"));
+        assertEquals(true, ok);
+
+        //EQUALS // Make sure order of input doesn't affect equals
+        WeightedSet<StringFieldValue> wset2 = new WeightedSet<>(DataType.TAG);
+        wset2.put(new StringFieldValue("bananaa"), 51);
+        wset2.put(new StringFieldValue("ballooo"), 50);
+        assertEquals(wset, wset2);
+
+    }
+
+    public void testAssignDoesNotIgnoreSpecialProperties() {
+        DataType type = DataType.getWeightedSet(DataType.STRING);
+        WeightedSet<StringFieldValue> set = new WeightedSet<>(type);
+        set.put(new StringFieldValue("hello"), 5);
+        set.put(new StringFieldValue("aba"), 10);
+        assertEquals(2, set.size());
+        assertEquals(new Integer(5), set.get(new StringFieldValue("hello")));
+        assertEquals(new Integer(10), set.get(new StringFieldValue("aba")));
+
+        DataType type2 = DataType.getWeightedSet(DataType.STRING, true, true);
+        WeightedSet<StringFieldValue> set2 = new WeightedSet<>(type2);
+        set2.put(new StringFieldValue("hi"), 6);
+        set2.put(new StringFieldValue("bye"), 13);
+        set2.put(new StringFieldValue("see you"), 15);
+        assertEquals(3, set2.size());
+        assertEquals(new Integer(6), set2.get(new StringFieldValue("hi")));
+        assertEquals(new Integer(13), set2.get(new StringFieldValue("bye")));
+        assertEquals(new Integer(15), set2.get(new StringFieldValue("see you")));
+
+        try {
+            set.assign(set2);
+            fail("it shouldn't be possible to assign a weighted set to another when types differ");
+        } catch (IllegalArgumentException iae) {
+            //success
+        }
+
+        assertEquals(2, set.size());
+        assertEquals(new Integer(5), set.get(new StringFieldValue("hello")));
+        assertEquals(new Integer(10), set.get(new StringFieldValue("aba")));
+    }
+
+    public void testWrappedMap() {
+        WeightedSet<StringFieldValue> ws = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+        Map<String, Integer> map = new LinkedHashMap<>();
+        map.put("foo", 1);
+        map.put("bar", 2);
+
+        ws.assign(map);
+
+        assertEquals(2, ws.size());
+        assertEquals(2, map.size());
+
+        assertTrue(ws.containsKey(new StringFieldValue("foo")));
+        assertTrue(ws.containsKey(new StringFieldValue("bar")));
+        assertFalse(ws.containsKey(new StringFieldValue("babar")));
+
+        ws.put(new StringFieldValue("banana"), 55);
+
+        assertEquals(3, ws.size());
+        assertEquals(3, map.size());
+
+        assertTrue(ws.containsValue(55));
+        assertFalse(ws.isEmpty());
+        ws.clear();
+        assertEquals(0, ws.size());
+        assertEquals(0, map.size());
+        assertTrue(map.isEmpty());
+        assertTrue(ws.isEmpty());
+
+        Map<StringFieldValue, Integer> tmp = new LinkedHashMap<>();
+        tmp.put(new StringFieldValue("cocacola"), 999);
+        tmp.put(new StringFieldValue("pepsicola"), 99999);
+        ws.putAll(tmp);
+
+        assertEquals(2, ws.size());
+        assertEquals(2, map.size());
+
+        ws.remove(new StringFieldValue("cocacola"));
+
+        assertEquals(1, ws.size());
+        assertEquals(1, map.size());
+
+        assertTrue(ws.contains(new StringFieldValue("pepsicola")));
+
+        ws.put(new StringFieldValue("solo"), 4);
+
+        assertEquals(2, ws.size());
+        assertEquals(2, map.size());
+
+        ws.add(new StringFieldValue("sitronbrus"));
+
+        assertEquals(3, ws.size());
+        assertEquals(3, map.size());
+
+        assertEquals(new Integer(1), ws.get(new StringFieldValue("sitronbrus")));
+    }
+
+    public void testAssigningWrappedSetToMapFieldValue() {
+        WeightedSet<StringFieldValue> weightedSet = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+        WeightedSet<StringFieldValue> assignmentTarget = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+        Map<String, Integer> rawMap = new LinkedHashMap<>();
+        rawMap.put("foo", 1);
+        rawMap.put("bar", 2);
+        weightedSet.assign(rawMap);
+        assignmentTarget.assign(weightedSet);
+        assertEquals(2, assignmentTarget.size());
+        assertEquals(Integer.valueOf(1), assignmentTarget.get(new StringFieldValue("foo")));
+        assertEquals(Integer.valueOf(2), assignmentTarget.get(new StringFieldValue("bar")));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/datatypes/blog.sd b/document/src/test/java/com/yahoo/document/datatypes/blog.sd
new file mode 100644
index 00000000000..1263c09ff95
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/blog.sd
@@ -0,0 +1,49 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search blog {
+
+  document blog {
+
+    field title type string {
+	    header
+        indexing: summary | index
+	    # index-to: default
+    }
+
+    field author type string {
+	    header
+	    indexing: summary | index
+	    # index-to: default
+    }
+
+    field body type string {
+        header
+	    indexing: summary | index
+    }
+
+    field url type uri {
+        header
+	    indexing: index | summary
+	    # index-to: default
+    }
+  }
+
+  annotation industry {
+    field vertical type string {}
+  }
+
+  annotation company inherits industry {
+    field name type string {}
+	field ceo type string {}	
+	field lat type long {}
+	field lon type long {}
+  }
+
+  annotation person {
+	field name type string {}
+  }
+
+  annotation location {
+  	field name type string {}
+  }
+}
+
diff --git a/document/src/test/java/com/yahoo/document/datatypes/documentmanager.blog.sd b/document/src/test/java/com/yahoo/document/datatypes/documentmanager.blog.sd
new file mode 100644
index 00000000000..313e8a88a3b
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/datatypes/documentmanager.blog.sd
@@ -0,0 +1,127 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+enablecompression false
+datatype[7]
+datatype[0].id -945638949
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name "blog.header"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[4]
+datatype[0].structtype[0].field[0].name "title"
+datatype[0].structtype[0].field[0].datatype 2
+datatype[0].structtype[0].field[0].id[0]
+datatype[0].structtype[0].field[1].name "author"
+datatype[0].structtype[0].field[1].datatype 2
+datatype[0].structtype[0].field[1].id[0]
+datatype[0].structtype[0].field[2].name "body"
+datatype[0].structtype[0].field[2].datatype 2
+datatype[0].structtype[0].field[2].id[0]
+datatype[0].structtype[0].field[3].name "url"
+datatype[0].structtype[0].field[3].datatype 10
+datatype[0].structtype[0].field[3].id[0]
+datatype[0].structtype[0].inherits[0]
+datatype[0].documenttype[0]
+datatype[0].annotationreftype[0]
+datatype[1].id 1387420336
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name "blog.body"
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[0]
+datatype[1].structtype[0].inherits[0]
+datatype[1].documenttype[0]
+datatype[1].annotationreftype[0]
+datatype[2].id -1386162972
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].name "blog"
+datatype[2].documenttype[0].version 0
+datatype[2].documenttype[0].headerstruct -945638949
+datatype[2].documenttype[0].bodystruct 1387420336
+datatype[2].documenttype[0].inherits[0]
+datatype[2].annotationreftype[0]
+datatype[3].id 912259135
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name "annotation.industry"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[1]
+datatype[3].structtype[0].field[0].name "vertical"
+datatype[3].structtype[0].field[0].datatype 2
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].inherits[0]
+datatype[3].documenttype[0]
+datatype[3].annotationreftype[0]
+datatype[4].id -476092672
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name "annotation.company"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[4]
+datatype[4].structtype[0].field[0].name "name"
+datatype[4].structtype[0].field[0].datatype 2
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[1].name "ceo"
+datatype[4].structtype[0].field[1].datatype 2
+datatype[4].structtype[0].field[1].id[0]
+datatype[4].structtype[0].field[2].name "lat"
+datatype[4].structtype[0].field[2].datatype 4
+datatype[4].structtype[0].field[2].id[0]
+datatype[4].structtype[0].field[3].name "lon"
+datatype[4].structtype[0].field[3].datatype 4
+datatype[4].structtype[0].field[3].id[0]
+datatype[4].structtype[0].inherits[1]
+datatype[4].structtype[0].inherits[0].name "annotation.industry"
+datatype[4].structtype[0].inherits[0].version 0
+datatype[4].documenttype[0]
+datatype[4].annotationreftype[0]
+datatype[5].id -1466283082
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name "annotation.person"
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].field[1]
+datatype[5].structtype[0].field[0].name "name"
+datatype[5].structtype[0].field[0].datatype 2
+datatype[5].structtype[0].field[0].id[0]
+datatype[5].structtype[0].inherits[0]
+datatype[5].documenttype[0]
+datatype[5].annotationreftype[0]
+datatype[6].id 515587158
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[1]
+datatype[6].structtype[0].name "annotation.location"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[1]
+datatype[6].structtype[0].field[0].name "name"
+datatype[6].structtype[0].field[0].datatype 2
+datatype[6].structtype[0].field[0].id[0]
+datatype[6].structtype[0].inherits[0]
+datatype[6].documenttype[0]
+datatype[6].annotationreftype[0]
+annotationtype[4]
+annotationtype[0].name "person"
+annotationtype[0].id 609952424
+annotationtype[0].datatype -1466283082
+annotationtype[0].inherits[0]
+annotationtype[1].name "location"
+annotationtype[1].id -270471211
+annotationtype[1].datatype 515587158
+annotationtype[1].inherits[0]
+annotationtype[2].name "company"
+annotationtype[2].id 1278713514
+annotationtype[2].datatype -476092672
+annotationtype[2].inherits[1]
+annotationtype[2].inherits[0].id 9765800
+annotationtype[3].name "industry"
+annotationtype[3].id 9765800
+annotationtype[3].datatype 912259135
+annotationtype[3].inherits[0]
diff --git a/document/src/test/java/com/yahoo/document/declaration/.gitignore b/document/src/test/java/com/yahoo/document/declaration/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/declaration/.gitignore
diff --git a/document/src/test/java/com/yahoo/document/docindoc.sd b/document/src/test/java/com/yahoo/document/docindoc.sd
new file mode 100644
index 00000000000..83bc4254fb2
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/docindoc.sd
@@ -0,0 +1,7 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search docindoc {
+  document docindoc {
+    field name type string { header }
+    field content type string { body }
+  }
+}
diff --git a/document/src/test/java/com/yahoo/document/documentmanager.docindoc.cfg b/document/src/test/java/com/yahoo/document/documentmanager.docindoc.cfg
new file mode 100644
index 00000000000..3347c3127b5
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/documentmanager.docindoc.cfg
@@ -0,0 +1,42 @@
+enablecompression false
+datatype[7]
+datatype[0].id -1407012075
+datatype[0].structtype[1]
+datatype[0].structtype[0].name "outerdoc.body"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[1]
+datatype[0].structtype[0].field[0].datatype -2035324352
+datatype[0].structtype[0].field[0].name "innerdocuments"
+datatype[1].id -1686125086
+datatype[1].structtype[1]
+datatype[1].structtype[0].name "docindoc.header"
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[1]
+datatype[1].structtype[0].field[0].datatype 2
+datatype[1].structtype[0].field[0].name "name"
+datatype[2].id -2035324352
+datatype[2].arraytype[1]
+datatype[2].arraytype[0].datatype 1447635645
+datatype[3].id -2040625920
+datatype[3].structtype[1]
+datatype[3].structtype[0].name "outerdoc.header"
+datatype[3].structtype[0].version 0
+datatype[4].id 1447635645
+datatype[4].documenttype[1]
+datatype[4].documenttype[0].bodystruct 2030224503
+datatype[4].documenttype[0].headerstruct -1686125086
+datatype[4].documenttype[0].name "docindoc"
+datatype[4].documenttype[0].version 0
+datatype[5].id 1748635999
+datatype[5].documenttype[1]
+datatype[5].documenttype[0].bodystruct -1407012075
+datatype[5].documenttype[0].headerstruct -2040625920
+datatype[5].documenttype[0].name "outerdoc"
+datatype[5].documenttype[0].version 0
+datatype[6].id 2030224503
+datatype[6].structtype[1]
+datatype[6].structtype[0].name "docindoc.body"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].field[1]
+datatype[6].structtype[0].field[0].datatype 2
+datatype[6].structtype[0].field[0].name "content"
diff --git a/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java b/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java
new file mode 100644
index 00000000000..ef8346768ba
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java
@@ -0,0 +1,183 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.fieldset;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentTestCaseBase;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.FieldValue;
+import org.junit.Test;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for field sets
+ */
+public class FieldSetTestCase extends DocumentTestCaseBase {
+
+    @Test
+    public void testClone() throws Exception {
+        assertTrue(new AllFields().clone() instanceof AllFields);
+        assertTrue(new NoFields().clone() instanceof NoFields);
+        assertTrue(new HeaderFields().clone() instanceof HeaderFields);
+        assertTrue(new BodyFields().clone() instanceof BodyFields);
+        assertTrue(new DocIdOnly().clone() instanceof DocIdOnly);
+    }
+
+    @Test
+    public void testParsing() {
+        FieldSetRepo repo = new FieldSetRepo();
+
+        assertTrue(repo.parse(docMan, "[all]") instanceof AllFields);
+        assertTrue(repo.parse(docMan, "[none]") instanceof NoFields);
+        assertTrue(repo.parse(docMan, "[id]") instanceof DocIdOnly);
+        assertTrue(repo.parse(docMan, "[header]") instanceof HeaderFields);
+        assertTrue(repo.parse(docMan, "[body]") instanceof BodyFields);
+
+        FieldCollection collection = (FieldCollection)repo.parse(docMan, "testdoc:stringattr,intattr");
+        assertEquals(2, collection.size());
+    }
+
+    void assertContains(String str1, String str2) throws Exception {
+        FieldSetRepo repo = new FieldSetRepo();
+        FieldSet set1 = repo.parse(docMan, str1);
+        FieldSet set2 = repo.parse(docMan, str2);
+        assertTrue(set1.clone().contains(set2.clone()));
+    }
+
+    void assertNotContains(String str1, String str2) throws Exception {
+        FieldSetRepo repo = new FieldSetRepo();
+        FieldSet set1 = repo.parse(docMan, str1);
+        FieldSet set2 = repo.parse(docMan, str2);
+        assertFalse(set1.clone().contains(set2.clone()));
+    }
+
+    void assertError(String str) {
+        try {
+            new FieldSetRepo().parse(docMan, str);
+            assertTrue(false);
+        } catch (Exception e) {
+        }
+    }
+
+    @Test
+    public void testContains() throws Exception {
+        Field headerField = testDocType.getField("intattr");
+        Field bodyField = testDocType.getField("rawattr");
+
+        assertFalse(headerField.contains(testDocType.getField("byteattr")));
+        assertTrue(headerField.contains(testDocType.getField("intattr")));
+        assertFalse(headerField.contains(bodyField));
+        assertTrue(headerField.contains(new DocIdOnly()));
+        assertTrue(headerField.contains(new NoFields()));
+        assertFalse(headerField.contains(new AllFields()));
+        assertFalse(headerField.contains(new HeaderFields()));
+        assertFalse(headerField.contains(new BodyFields()));
+
+        assertFalse(new NoFields().contains(headerField));
+        assertFalse(new NoFields().contains(new AllFields()));
+        assertFalse(new NoFields().contains(new DocIdOnly()));
+
+        assertTrue(new AllFields().contains(new HeaderFields()));
+        assertTrue(new AllFields().contains(headerField));
+        assertTrue(new AllFields().contains(bodyField));
+        assertTrue(new AllFields().contains(new BodyFields()));
+        assertTrue(new AllFields().contains(new DocIdOnly()));
+        assertTrue(new AllFields().contains(new NoFields()));
+        assertTrue(new AllFields().contains(new AllFields()));
+
+        assertTrue(new DocIdOnly().contains(new NoFields()));
+        assertTrue(new DocIdOnly().contains(new DocIdOnly()));
+        assertFalse(new DocIdOnly().contains(headerField));
+
+        assertTrue(new HeaderFields().contains(headerField));
+        assertFalse(new HeaderFields().contains(bodyField));
+        assertTrue(new HeaderFields().contains(new DocIdOnly()));
+        assertTrue(new HeaderFields().contains(new NoFields()));
+
+        assertContains("[body]", "testdoc:rawattr");
+        assertContains("[header]", "testdoc:intattr");
+        assertNotContains("[header]", "testdoc:rawattr");
+        assertContains("testdoc:rawattr,intattr", "testdoc:intattr");
+        assertNotContains("testdoc:intattr", "testdoc:rawattr,intattr");
+        assertContains("testdoc:intattr,rawattr", "testdoc:rawattr,intattr");
+
+        assertError("nodoctype");
+        assertError("unknowndoctype:foo");
+        assertError("testdoc:unknownfield");
+        assertError("[badid]");
+    }
+
+    String stringifyFields(Document doc) {
+        String retVal = "";
+        for (Iterator<Map.Entry<Field, FieldValue>> i = doc.iterator(); i.hasNext(); ) {
+            Map.Entry<Field, FieldValue> v = i.next();
+
+            if (retVal.length() > 0) {
+                retVal += ",";
+            }
+            retVal += v.getKey().getName() + ":" + v.getValue().toString();
+        }
+
+        return retVal;
+    }
+
+    String doCopyFields(Document source, String fieldSet) {
+        FieldSetRepo repo = new FieldSetRepo();
+        Document target = new Document(source.getDataType(), source.getId());
+        repo.copyFields(source, target, repo.parse(docMan, fieldSet));
+        return stringifyFields(target);
+    }
+
+    @Test
+    public void testCopyDocumentFields() {
+        Document doc = getTestDocument();
+        doc.removeFieldValue("rawattr");
+
+        assertEquals("floatattr:3.56", doCopyFields(doc, "[body]"));
+        assertEquals("stringattr:tjohei,intattr:50,byteattr:30,floatattr:3.56", doCopyFields(doc, "[all]"));
+        assertEquals("stringattr:tjohei,intattr:50,byteattr:30", doCopyFields(doc, "[header]"));
+        assertEquals("byteattr:30,floatattr:3.56", doCopyFields(doc, "testdoc:floatattr,byteattr"));
+    }
+
+    String doStripFields(Document source, String fieldSet) {
+        FieldSetRepo repo = new FieldSetRepo();
+        Document target = source.clone();
+        repo.stripFields(target, repo.parse(docMan, fieldSet));
+        return stringifyFields(target);
+    }
+
+    @Test
+    public void testStripFields() {
+        Document doc = getTestDocument();
+        doc.removeFieldValue("rawattr");
+
+        assertEquals("floatattr:3.56", doStripFields(doc, "[body]"));
+        assertEquals("stringattr:tjohei,intattr:50,byteattr:30,floatattr:3.56", doStripFields(doc, "[all]"));
+        assertEquals("stringattr:tjohei,intattr:50,byteattr:30", doStripFields(doc, "[header]"));
+        assertEquals("byteattr:30,floatattr:3.56", doStripFields(doc, "testdoc:floatattr,byteattr"));
+    }
+
+    @Test
+    public void testSerialize() {
+        String fieldSets[] =
+                {
+                        "[all]",
+                        "[none]",
+                        "[header]",
+                        "[docid]",
+                        "[body]",
+                        "testdoc:rawattr",
+                        "testdoc:rawattr,intattr"
+                };
+
+        FieldSetRepo repo = new FieldSetRepo();
+        for (String fieldSet : fieldSets) {
+            assertEquals(fieldSet, repo.serialize(repo.parse(docMan, fieldSet)));
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
new file mode 100644
index 00000000000..860b2c9fba8
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
@@ -0,0 +1,1252 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.io.JsonStringEncoder;
+import com.google.common.base.Joiner;
+import com.yahoo.collections.Tuple2;
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentOperation;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.Field;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.MapFieldValue;
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.document.update.*;
+import com.yahoo.document.update.ArithmeticValueUpdate.Operator;
+import com.yahoo.tensor.MapTensor;
+import com.yahoo.text.Utf8;
+import org.apache.commons.codec.binary.Base64;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.mockito.internal.matchers.Contains;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.*;
+
+/**
+ * Basic test of JSON streams to Vespa document instances.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author vegard
+ */
+public class JsonReaderTestCase {
+    DocumentTypeManager types;
+    JsonFactory parserFactory;
+
+    @Rule
+    public ExpectedException exception = ExpectedException.none();
+
+    @Before
+    public void setUp() throws Exception {
+        parserFactory = new JsonFactory();
+        types = new DocumentTypeManager();
+        {
+            DocumentType x = new DocumentType("smoke");
+            x.addField(new Field("something", DataType.STRING));
+            x.addField(new Field("nalle", DataType.STRING));
+            x.addField(new Field("int1", DataType.INT));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("mirrors");
+            StructDataType woo = new StructDataType("woo");
+            woo.addField(new Field("sandra", DataType.STRING));
+            woo.addField(new Field("cloud", DataType.STRING));
+            x.addField(new Field("skuggsjaa", woo));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testarray");
+            DataType d = new ArrayDataType(DataType.STRING);
+            x.addField(new Field("actualarray", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testset");
+            DataType d = new WeightedSetDataType(DataType.STRING, true, true);
+            x.addField(new Field("actualset", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testmap");
+            DataType d = new MapDataType(DataType.STRING, DataType.STRING);
+            x.addField(new Field("actualmap", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testraw");
+            DataType d = DataType.RAW;
+            x.addField(new Field("actualraw", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testMapStringToArrayOfInt");
+            DataType value = new ArrayDataType(DataType.INT);
+            DataType d = new MapDataType(DataType.STRING, value);
+            x.addField(new Field("actualMapStringToArrayOfInt", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testsinglepos");
+            DataType d = PositionDataType.INSTANCE;
+            x.addField(new Field("singlepos", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testtensor");
+            x.addField(new Field("tensorfield", DataType.TENSOR));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testpredicate");
+            x.addField(new Field("boolean", DataType.PREDICATE));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testint");
+            x.addField(new Field("integerfield", DataType.INT));
+            types.registerDocumentType(x);
+        }
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        types = null;
+        parserFactory = null;
+        exception = ExpectedException.none();
+    }
+
+    @Test
+    public final void readSingleDocumentPut() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        DocumentPut put = (DocumentPut) r.readSingleDocument(JsonReader.SupportedOperation.PUT, "id:unittest:smoke::whee");
+        smokeTestDoc(put.getDocument());
+    }
+
+    @Test
+    public final void readSingleDocumentUpdate() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"something\": {"
+                        + " \"assign\": \"orOther\" }}" + " }"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        DocumentUpdate doc = (DocumentUpdate) r.readSingleDocument(JsonReader.SupportedOperation.UPDATE, "id:unittest:smoke::whee");
+        FieldUpdate f = doc.getFieldUpdate("something");
+        assertEquals(1, f.size());
+        assertTrue(f.getValueUpdate(0) instanceof AssignValueUpdate);
+    }
+
+    @Test
+    public final void readClearField() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"int1\": {"
+                        + " \"assign\": null }}" + " }"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        DocumentUpdate doc = (DocumentUpdate) r.readSingleDocument(JsonReader.SupportedOperation.UPDATE, "id:unittest:smoke::whee");
+        FieldUpdate f = doc.getFieldUpdate("int1");
+        assertEquals(1, f.size());
+        assertTrue(f.getValueUpdate(0) instanceof ClearValueUpdate);
+        assertNull(f.getValueUpdate(0).getValue());
+    }
+
+
+    @Test
+    public final void smokeTest() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        smokeTestDoc(put.getDocument());
+    }
+
+    @Test
+    public final void docIdLookaheadTest() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{"
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"},"
+                        + "\"put\": \"id:unittest:smoke::whee\""
+                        + "}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        smokeTestDoc(put.getDocument());
+    }
+
+
+    @Test
+    public final void emptyDocTest() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": {}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        assertEquals("id:unittest:smoke::whee", parseInfo.documentId.toString());
+    }
+
+    @Test
+    public final void testStruct() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:mirrors::whee\","
+                        + " \"fields\": { "
+                        + "\"skuggsjaa\": {"
+                        + "\"sandra\": \"person\","
+                        + " \"cloud\": \"another person\"}}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue(doc.getField("skuggsjaa"));
+        assertSame(Struct.class, f.getClass());
+        Struct s = (Struct) f;
+        assertEquals("person", ((StringFieldValue) s.getFieldValue("sandra")).getString());
+    }
+
+    @Test
+    public final void testUpdateArray() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:testarray::whee\","
+                        + " \"fields\": { " + "\"actualarray\": {"
+                        + " \"add\": ["
+                        + " \"person\","
+                        + " \"another person\"]}}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+        r.readUpdate(doc);
+        checkSimpleArrayAdd(doc);
+    }
+
+    @Test
+    public final void testUpdateWeighted() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:testset::whee\","
+                        + " \"fields\": { " + "\"actualset\": {"
+                        + " \"add\": {"
+                        + " \"person\": 37,"
+                        + " \"another person\": 41}}}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+        r.readUpdate(doc);
+        Map<String, Integer> weights = new HashMap<>();
+        FieldUpdate x = doc.getFieldUpdate("actualset");
+        for (ValueUpdate<?> v : x.getValueUpdates()) {
+            AddValueUpdate adder = (AddValueUpdate) v;
+            final String s = ((StringFieldValue) adder.getValue()).getString();
+            weights.put(s, adder.getWeight());
+        }
+        assertEquals(2, weights.size());
+        final String o = "person";
+        final String o2 = "another person";
+        assertTrue(weights.containsKey(o));
+        assertTrue(weights.containsKey(o2));
+        assertEquals(Integer.valueOf(37), weights.get(o));
+        assertEquals(Integer.valueOf(41), weights.get(o2));
+    }
+
+    @Test
+    public final void testUpdateMatch() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:testset::whee\","
+                        + " \"fields\": { " + "\"actualset\": {"
+                        + " \"match\": {"
+                        + " \"element\": \"person\","
+                        + " \"increment\": 13}}}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+
+        r.readUpdate(doc);
+        Map<String, Tuple2<Number, String>> matches = new HashMap<>();
+        FieldUpdate x = doc.getFieldUpdate("actualset");
+        for (ValueUpdate<?> v : x.getValueUpdates()) {
+            MapValueUpdate adder = (MapValueUpdate) v;
+            final String key = ((StringFieldValue) adder.getValue())
+                    .getString();
+            String op = ((ArithmeticValueUpdate) adder.getUpdate())
+                    .getOperator().toString();
+            Number n = ((ArithmeticValueUpdate) adder.getUpdate()).getOperand();
+            matches.put(key, new Tuple2<>(n, op));
+        }
+        assertEquals(1, matches.size());
+        final String o = "person";
+        assertEquals("ADD", matches.get(o).second);
+        assertEquals(Double.valueOf(13), matches.get(o).first);
+    }
+
+    @SuppressWarnings({ "cast", "unchecked", "rawtypes" })
+    @Test
+    public final void testArithmeticOperators() {
+        Tuple2[] operations = new Tuple2[] {
+                new Tuple2<String, Operator>(JsonReader.UPDATE_DECREMENT,
+                        ArithmeticValueUpdate.Operator.SUB),
+                new Tuple2<String, Operator>(JsonReader.UPDATE_DIVIDE,
+                        ArithmeticValueUpdate.Operator.DIV),
+                new Tuple2<String, Operator>(JsonReader.UPDATE_INCREMENT,
+                        ArithmeticValueUpdate.Operator.ADD),
+                new Tuple2<String, Operator>(JsonReader.UPDATE_MULTIPLY,
+                        ArithmeticValueUpdate.Operator.MUL) };
+        for (Tuple2<String, Operator> operator : operations) {
+            InputStream rawDoc = new ByteArrayInputStream(
+                    Utf8.toBytes("{\"update\": \"id:unittest:testset::whee\","
+                            + " \"fields\": { " + "\"actualset\": {"
+                            + " \"match\": {" + " \"element\": \"person\","
+                            + " \"" + (String) operator.first + "\": 13}}}}"));
+
+            JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+            JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+            DocumentType docType = r.readDocumentType(parseInfo.documentId);
+            DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+
+            r.readUpdate(doc);
+            Map<String, Tuple2<Number, Operator>> matches = new HashMap<>();
+            FieldUpdate x = doc.getFieldUpdate("actualset");
+            for (ValueUpdate v : x.getValueUpdates()) {
+                MapValueUpdate adder = (MapValueUpdate) v;
+                final String key = ((StringFieldValue) adder.getValue())
+                        .getString();
+                Operator op = ((ArithmeticValueUpdate) adder
+                        .getUpdate()).getOperator();
+                Number n = ((ArithmeticValueUpdate) adder.getUpdate())
+                        .getOperand();
+                matches.put(key, new Tuple2<>(n, op));
+            }
+            assertEquals(1, matches.size());
+            final String o = "person";
+            assertSame(operator.second, matches.get(o).second);
+            assertEquals(Double.valueOf(13), matches.get(o).first);
+        }
+    }
+
+    @SuppressWarnings("rawtypes")
+    @Test
+    public final void testArrayIndexing() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:testarray::whee\","
+                        + " \"fields\": { " + "\"actualarray\": {"
+                        + " \"match\": {"
+                        + " \"element\": 3,"
+                        + " \"assign\": \"nalle\"}}}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+
+        r.readUpdate(doc);
+        Map<Number, String> matches = new HashMap<>();
+        FieldUpdate x = doc.getFieldUpdate("actualarray");
+        for (ValueUpdate v : x.getValueUpdates()) {
+            MapValueUpdate adder = (MapValueUpdate) v;
+            final Number key = ((IntegerFieldValue) adder.getValue())
+                    .getNumber();
+            String op = ((StringFieldValue) ((AssignValueUpdate) adder.getUpdate())
+                    .getValue()).getString();
+            matches.put(key, op);
+        }
+        assertEquals(1, matches.size());
+        Number n = Integer.valueOf(3);
+        assertEquals("nalle", matches.get(n));
+    }
+
+    @Test
+    public final void testDocumentRemove() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"remove\": \"id:unittest:smoke::whee\""
+                        + " }}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        DocumentType docType = r.readDocumentType(new DocumentId("id:unittest:smoke::whee"));
+        assertEquals("smoke", docType.getName());
+    }
+
+    @Test
+    public final void testWeightedSet() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:testset::whee\","
+                        + " \"fields\": { \"actualset\": {"
+                        + " \"nalle\": 2,"
+                        + " \"tralle\": 7 }}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue(doc.getField("actualset"));
+        assertSame(WeightedSet.class, f.getClass());
+        WeightedSet<?> w = (WeightedSet<?>) f;
+        assertEquals(2, w.size());
+        assertEquals(new Integer(2), w.get(new StringFieldValue("nalle")));
+        assertEquals(new Integer(7), w.get(new StringFieldValue("tralle")));
+    }
+
+    @Test
+    public final void testArray() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:testarray::whee\","
+                        + " \"fields\": { \"actualarray\": ["
+                        + " \"nalle\","
+                        + " \"tralle\"]}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue(doc.getField("actualarray"));
+        assertSame(Array.class, f.getClass());
+        Array<?> a = (Array<?>) f;
+        assertEquals(2, a.size());
+        assertEquals(new StringFieldValue("nalle"), a.get(0));
+        assertEquals(new StringFieldValue("tralle"), a.get(1));
+    }
+
+    @Test
+    public final void testMap() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:testmap::whee\","
+                        + " \"fields\": { \"actualmap\": ["
+                        + " { \"key\": \"nalle\", \"value\": \"kalle\"},"
+                        + " { \"key\": \"tralle\", \"value\": \"skalle\"} ]}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue(doc.getField("actualmap"));
+        assertSame(MapFieldValue.class, f.getClass());
+        MapFieldValue<?, ?> m = (MapFieldValue<?, ?>) f;
+        assertEquals(2, m.size());
+        assertEquals(new StringFieldValue("kalle"), m.get(new StringFieldValue("nalle")));
+        assertEquals(new StringFieldValue("skalle"), m.get(new StringFieldValue("tralle")));
+    }
+
+    @Test
+    public final void testPositionPositive() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:testsinglepos::bamf\","
+                        + " \"fields\": { \"singlepos\": \"N63.429722;E10.393333\" }}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue(doc.getField("singlepos"));
+        assertSame(Struct.class, f.getClass());
+        assertEquals(10393333, PositionDataType.getXValue(f).getInteger());
+        assertEquals(63429722, PositionDataType.getYValue(f).getInteger());
+    }
+
+    @Test
+    public final void testPositionNegative() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:testsinglepos::bamf\","
+                        + " \"fields\": { \"singlepos\": \"W46.63;S23.55\" }}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue(doc.getField("singlepos"));
+        assertSame(Struct.class, f.getClass());
+        assertEquals(-46630000, PositionDataType.getXValue(f).getInteger());
+        assertEquals(-23550000, PositionDataType.getYValue(f).getInteger());
+    }
+
+    @Test
+    public final void testRaw() {
+        String stuff = new String(new JsonStringEncoder().quoteAsString(new Base64().encodeToString(Utf8.toBytes("smoketest"))));
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:testraw::whee\","
+                        + " \"fields\": { \"actualraw\": \""
+                        + stuff
+                        + "\""
+                        + " }}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue(doc.getField("actualraw"));
+        assertSame(Raw.class, f.getClass());
+        Raw s = (Raw) f;
+        ByteBuffer b = s.getByteBuffer();
+        assertEquals("smoketest", Utf8.toString(b));
+    }
+
+    @Test
+    public final void testMapStringToArrayOfInt() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:testMapStringToArrayOfInt::whee\","
+                        + " \"fields\": { \"actualMapStringToArrayOfInt\": ["
+                        + "{ \"key\": \"bamse\", \"value\": [1, 2, 3] }"
+                        + "]}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        Document doc = put.getDocument();
+        FieldValue f = doc.getFieldValue("actualMapStringToArrayOfInt");
+        assertSame(MapFieldValue.class, f.getClass());
+        MapFieldValue<?, ?> m = (MapFieldValue<?, ?>) f;
+        Array<?> a = (Array<?>) m.get(new StringFieldValue("bamse"));
+        assertEquals(3, a.size());
+        assertEquals(new IntegerFieldValue(1), a.get(0));
+        assertEquals(new IntegerFieldValue(2), a.get(1));
+        assertEquals(new IntegerFieldValue(3), a.get(2));
+    }
+
+    @Test
+    public final void testAssignToString() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"something\": {"
+                        + " \"assign\": \"orOther\" }}" + " }"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+        r.readUpdate(doc);
+        FieldUpdate f = doc.getFieldUpdate("something");
+        assertEquals(1, f.size());
+        AssignValueUpdate a = (AssignValueUpdate) f.getValueUpdate(0);
+        assertEquals(new StringFieldValue("orOther"), a.getValue());
+    }
+
+    @Test
+    public final void testAssignToArray() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:testMapStringToArrayOfInt::whee\","
+                        + " \"fields\": { \"actualMapStringToArrayOfInt\": {"
+                        + " \"assign\": ["
+                        + "{ \"key\": \"bamse\", \"value\": [1, 2, 3] }"
+                        + "]}}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+        r.readUpdate(doc);
+        FieldUpdate f = doc.getFieldUpdate("actualMapStringToArrayOfInt");
+        assertEquals(1, f.size());
+        AssignValueUpdate assign = (AssignValueUpdate) f.getValueUpdate(0);
+        MapFieldValue<?, ?> m = (MapFieldValue<?, ?>) assign.getValue();
+        Array<?> a = (Array<?>) m.get(new StringFieldValue("bamse"));
+        assertEquals(3, a.size());
+        assertEquals(new IntegerFieldValue(1), a.get(0));
+        assertEquals(new IntegerFieldValue(2), a.get(1));
+        assertEquals(new IntegerFieldValue(3), a.get(2));
+    }
+
+    @Test
+    public final void testAssignToWeightedSet() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"update\": \"id:unittest:testset::whee\","
+                        + " \"fields\": { " + "\"actualset\": {"
+                        + " \"assign\": {"
+                        + " \"person\": 37,"
+                        + " \"another person\": 41}}}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId);
+        r.readUpdate(doc);
+        FieldUpdate x = doc.getFieldUpdate("actualset");
+        assertEquals(1, x.size());
+        AssignValueUpdate assign = (AssignValueUpdate) x.getValueUpdate(0);
+        WeightedSet<?> w = (WeightedSet<?>) assign.getValue();
+        assertEquals(2, w.size());
+        assertEquals(new Integer(37), w.get(new StringFieldValue("person")));
+        assertEquals(new Integer(41), w.get(new StringFieldValue("another person")));
+    }
+
+
+    @Test
+    public final void testCompleteFeed() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("[{\"put\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"}}" + ", "
+                        + "{\"update\": \"id:unittest:testarray::whee\","
+                        + " \"fields\": { " + "\"actualarray\": {"
+                        + " \"add\": [" + " \"person\","
+                        + " \"another person\"]}}}" + ", "
+                        + "{\"remove\": \"id:unittest:smoke::whee\"}]"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+
+        controlBasicFeed(r);
+    }
+
+    @Test
+    public final void testCompleteFeedWithCreateAndCondition() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("[{\"put\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"}}" + ", "
+                        + "{"
+                        +  "\"condition\":\"bla\","
+                        +  "\"update\": \"id:unittest:testarray::whee\","
+                        + " \"create\":true,"
+                        + " \"fields\": { " + "\"actualarray\": {"
+                        + " \"add\": [" + " \"person\","
+                        + " \"another person\"]}}}" + ", "
+                        + "{\"remove\": \"id:unittest:smoke::whee\"}]"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+
+        DocumentOperation d = r.next();
+        Document doc = ((DocumentPut) d).getDocument();
+        smokeTestDoc(doc);
+
+        d = r.next();
+        DocumentUpdate update = (DocumentUpdate) d;
+        checkSimpleArrayAdd(update);
+        assertThat(update.getCreateIfNonExistent(), is(true));
+        assertThat(update.getCondition().getSelection(), is("bla"));
+
+        d = r.next();
+        DocumentRemove remove = (DocumentRemove) d;
+        assertEquals("smoke", remove.getId().getDocType());
+
+        assertNull(r.next());
+    }
+
+    @Test
+    public final void testUpdateWithConditionAndCreateInDifferentOrdering() {
+        final int  documentsCreated = 106;
+        List<String> parts = Arrays.asList(
+                "\"condition\":\"bla\"",
+                "\"update\": \"id:unittest:testarray::whee\"",
+                " \"fields\": { " + "\"actualarray\": { \"add\": [" + " \"person\",\"another person\"]}}",
+                " \"create\":true");
+        final Random random = new Random(42);
+        StringBuilder documents = new StringBuilder("[");
+        for (int x = 0; x < documentsCreated; x++) {
+            Collections.shuffle(parts, random);
+            documents.append("{").append(Joiner.on(",").join(parts)).append("}");
+            if (x < documentsCreated -1) {
+                documents.append(",");
+            }
+        }
+        documents.append("]");
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes(documents.toString()));
+
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+
+        for (int x = 0; x < documentsCreated; x++) {
+            DocumentUpdate update = (DocumentUpdate) r.next();
+            checkSimpleArrayAdd(update);
+            assertThat(update.getCreateIfNonExistent(), is(true));
+            assertThat(update.getCondition().getSelection(), is("bla"));
+
+        }
+
+        assertNull(r.next());
+    }
+
+
+    @Test(expected=RuntimeException.class)
+    public final void testCreateIfNonExistentInPut() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("[{"
+                        + " \"create\":true,"
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"},"
+                        + "\"put\": \"id:unittest:smoke::whee\""
+                        + "}]"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        r.next();
+    }
+
+    @Test
+    public final void testCompleteFeedWithIdAfterFields() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("[{"
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"},"
+                        + "\"put\": \"id:unittest:smoke::whee\""
+                        + "}" + ", "
+                        + "{"
+                        + " \"fields\": { " + "\"actualarray\": {"
+                        + " \"add\": [" + " \"person\","
+                        + " \"another person\"]}},"
+                        + "\"update\": \"id:unittest:testarray::whee\""
+                        + "}" + ", "
+                        + "{\"remove\": \"id:unittest:smoke::whee\"}]"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+
+        controlBasicFeed(r);
+    }
+
+    protected void controlBasicFeed(JsonReader r) {
+        DocumentOperation d = r.next();
+        Document doc = ((DocumentPut) d).getDocument();
+        smokeTestDoc(doc);
+
+        d = r.next();
+        DocumentUpdate update = (DocumentUpdate) d;
+        checkSimpleArrayAdd(update);
+
+        d = r.next();
+        DocumentRemove remove = (DocumentRemove) d;
+        assertEquals("smoke", remove.getId().getDocType());
+
+        assertNull(r.next());
+    }
+
+
+    @Test
+    public final void testCompleteFeedWithEmptyDoc() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("[{\"put\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": {}}" + ", "
+                        + "{\"update\": \"id:unittest:testarray::whee\","
+                        + " \"fields\": {}}" + ", "
+                        + "{\"remove\": \"id:unittest:smoke::whee\"}]"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+
+
+        DocumentOperation d = r.next();
+        Document doc = ((DocumentPut) d).getDocument();
+        assertEquals("smoke", doc.getId().getDocType());
+
+        d = r.next();
+        DocumentUpdate update = (DocumentUpdate) d;
+        assertEquals("testarray", update.getId().getDocType());
+
+        d = r.next();
+        DocumentRemove remove = (DocumentRemove) d;
+        assertEquals("smoke", remove.getId().getDocType());
+
+        assertNull(r.next());
+
+    }
+
+    private void checkSimpleArrayAdd(DocumentUpdate update) {
+        Set<String> toAdd = new HashSet<>();
+        FieldUpdate x = update.getFieldUpdate("actualarray");
+        for (ValueUpdate<?> v : x.getValueUpdates()) {
+            AddValueUpdate adder = (AddValueUpdate) v;
+            toAdd.add(((StringFieldValue) adder.getValue()).getString());
+        }
+        assertEquals(2, toAdd.size());
+        assertTrue(toAdd.contains("person"));
+        assertTrue(toAdd.contains("another person"));
+    }
+
+    private void smokeTestDoc(Document doc) {
+        FieldValue f = doc.getFieldValue(doc.getField("nalle"));
+        assertSame(StringFieldValue.class, f.getClass());
+        StringFieldValue s = (StringFieldValue) f;
+        assertEquals("bamse", s.getString());
+    }
+
+    @Test
+    public final void misspelledFieldTest() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"smething\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        exception.expect(NullPointerException.class);
+        exception.expectMessage("Could not get field \"smething\" in the structure of type \"smoke\".");
+        r.readPut(put);
+    }
+
+    @Test
+    public final void feedWithBasicErrorTest() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("["
+                        + "  { \"put\": \"id:test:smoke::0\", \"fields\": { \"something\": \"foo\" } },"
+                        + "  { \"put\": \"id:test:smoke::1\", \"fields\": { \"something\": \"foo\" } },"
+                        + "  { \"put\": \"id:test:smoke::2\", \"fields\": { \"something\": \"foo\" } },"
+                        + "]"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        exception.expect(RuntimeException.class);
+        exception.expectMessage("JsonParseException");
+        while (r.next() != null);
+    }
+
+    @Test
+    public final void idAsAliasForPutTest() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("{\"id\": \"id:unittest:smoke::whee\","
+                        + " \"fields\": { \"something\": \"smoketest\","
+                        + " \"nalle\": \"bamse\"}}"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(parseInfo.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
+        r.readPut(put);
+        smokeTestDoc(put.getDocument());
+    }
+
+    private void testFeedWithTestAndSetCondition(String jsonDoc) {
+        final ByteArrayInputStream parseInfoDoc = new ByteArrayInputStream(Utf8.toBytes(jsonDoc));
+        final JsonReader reader = new JsonReader(types, parseInfoDoc, parserFactory);
+        final int NUM_OPERATIONS_IN_FEED = 3;
+
+        for (int i = 0; i < NUM_OPERATIONS_IN_FEED; i++) {
+            DocumentOperation operation = reader.next();
+
+            assertTrue("A test and set condition should be present",
+                    operation.getCondition().isPresent());
+
+            assertEquals("DocumentOperation's test and set condition should be equal to the one in the JSON feed",
+                    "smoke.something == \"smoketest\"",
+                    operation.getCondition().getSelection());
+        }
+
+        assertNull(reader.next());
+    }
+
+    @Test
+    public final void testFeedWithTestAndSetConditionOrderingOne() {
+        testFeedWithTestAndSetCondition(
+                inputJson("[",
+                        "      {",
+                        "          'put': 'id:unittest:smoke::whee',",
+                        "          'condition': 'smoke.something == \\'smoketest\\'',",
+                        "          'fields': {",
+                        "              'something': 'smoketest',",
+                        "              'nalle': 'bamse'",
+                        "          }",
+                        "      },",
+                        "      {",
+                        "          'update': 'id:unittest:testarray::whee',",
+                        "          'condition': 'smoke.something == \\'smoketest\\'',",
+                        "          'fields': {",
+                        "              'actualarray': {",
+                        "                  'add': [",
+                        "                      'person',",
+                        "                      'another person'",
+                        "                   ]",
+                        "              }",
+                        "          }",
+                        "      },",
+                        "      {",
+                        "          'remove': 'id:unittest:smoke::whee',",
+                        "          'condition': 'smoke.something == \\'smoketest\\''",
+                        "      }",
+                        "]"
+                ));
+    }
+
+    @Test
+    public final void testFeedWithTestAndSetConditionOrderingTwo() {
+        testFeedWithTestAndSetCondition(
+                inputJson("[",
+                        "      {",
+                        "          'condition': 'smoke.something == \\'smoketest\\'',",
+                        "          'put': 'id:unittest:smoke::whee',",
+                        "          'fields': {",
+                        "              'something': 'smoketest',",
+                        "              'nalle': 'bamse'",
+                        "          }",
+                        "      },",
+                        "      {",
+                        "          'condition': 'smoke.something == \\'smoketest\\'',",
+                        "          'update': 'id:unittest:testarray::whee',",
+                        "          'fields': {",
+                        "              'actualarray': {",
+                        "                  'add': [",
+                        "                      'person',",
+                        "                      'another person'",
+                        "                   ]",
+                        "              }",
+                        "          }",
+                        "      },",
+                        "      {",
+                        "          'condition': 'smoke.something == \\'smoketest\\'',",
+                        "          'remove': 'id:unittest:smoke::whee'",
+                        "      }",
+                        "]"
+                ));
+    }
+
+    @Test
+    public final void testFeedWithTestAndSetConditionOrderingThree() {
+        testFeedWithTestAndSetCondition(
+                inputJson("[",
+                        "      {",
+                        "          'put': 'id:unittest:smoke::whee',",
+                        "          'fields': {",
+                        "              'something': 'smoketest',",
+                        "              'nalle': 'bamse'",
+                        "          },",
+                        "          'condition': 'smoke.something == \\'smoketest\\''",
+                        "      },",
+                        "      {",
+                        "          'update': 'id:unittest:testarray::whee',",
+                        "          'fields': {",
+                        "              'actualarray': {",
+                        "                  'add': [",
+                        "                      'person',",
+                        "                      'another person'",
+                        "                   ]",
+                        "              }",
+                        "          },",
+                        "          'condition': 'smoke.something == \\'smoketest\\''",
+                        "      },",
+                        "      {",
+                        "          'remove': 'id:unittest:smoke::whee',",
+                        "          'condition': 'smoke.something == \\'smoketest\\''",
+                        "      }",
+                        "]"
+                ));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testInvalidFieldAfterFieldsFieldShouldFailParse() {
+        final String jsonData = inputJson(
+                "[",
+                "      {",
+                "          'put': 'id:unittest:smoke::whee',",
+                "          'fields': {",
+                "              'something': 'smoketest',",
+                "              'nalle': 'bamse'",
+                "          },",
+                "          'bjarne': 'stroustrup'",
+                "      }",
+                "]");
+
+        new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next();
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testInvalidFieldBeforeFieldsFieldShouldFailParse() {
+        final String jsonData = inputJson(
+                "[",
+                "      {",
+                "          'update': 'id:unittest:testarray::whee',",
+                "          'what is this': 'nothing to see here',",
+                "          'fields': {",
+                "              'actualarray': {",
+                "                  'add': [",
+                "                      'person',",
+                "                      'another person'",
+                "                   ]",
+                "              }",
+                "          }",
+                "      }",
+                "]");
+
+        new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next();
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testInvalidFieldWithoutFieldsFieldShouldFailParse() {
+        final String jsonData = inputJson(
+                "[",
+                "      {",
+                "          'remove': 'id:unittest:smoke::whee',",
+                "          'what is love': 'baby, do not hurt me... much'",
+                "      }",
+                "]");
+
+        new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next();
+    }
+
+    static ByteArrayInputStream jsonToInputStream(String json) {
+        return new ByteArrayInputStream(Utf8.toBytes(json));
+    }
+
+    /**
+     * Convenience method to input JSON without escaping double quotes and newlines
+     * Each parameter represents a line of JSON encoded data
+     * The lines are joined with newline and single quotes are replaced with double quotes
+     */
+    static String inputJson(String... lines) {
+        return Joiner.on("\n").join(lines).replaceAll("'", "\"");
+    }
+
+    @Test
+    public void testParsingWithoutTensorField() {
+        Document doc = createPutWithoutTensor().getDocument();
+        assertEquals("testtensor", doc.getId().getDocType());
+        assertEquals("id:unittest:testtensor::0", doc.getId().toString());
+        TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("tensorfield"));
+        assertNull(fieldValue);
+    }
+
+    @Test
+    public void testParsingOfEmptyTensor() {
+        assertTensorField("{}", createPutWithTensor("{}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithEmptyDimensions() {
+        assertTensorField("{}",
+                createPutWithTensor("{ "
+                        + "  \"dimensions\": [] "
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithEmptyCells() {
+        assertTensorField("{}",
+                createPutWithTensor("{ "
+                        + "  \"cells\": [] "
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithDimensions() {
+        assertTensorField("( {{x:-,y:-}:1.0} * {} )",
+                createPutWithTensor("{ "
+                        + "  \"dimensions\": [\"x\",\"y\"] "
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithCells() {
+        assertTensorField("{{x:a,y:b}:2.0,{x:c}:3.0}}",
+                createPutWithTensor("{ "
+                        + "  \"cells\": [ "
+                        + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+                        + "      \"value\": 2.0 }, "
+                        + "    { \"address\": { \"x\": \"c\" }, "
+                        + "      \"value\": 3.0 } "
+                        + "  ]"
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithSingleCellInDifferentJsonOrder() {
+        assertTensorField("{{x:a,y:b}:2.0}",
+                createPutWithTensor("{ "
+                        + "  \"cells\": [ "
+                        + "    { \"value\": 2.0, "
+                        + "      \"address\": { \"x\": \"a\", \"y\": \"b\" } } "
+                        + "  ]"
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithSingleCellWithoutAddress() {
+        assertTensorField("{{}:2.0}",
+                createPutWithTensor("{ "
+                        + "  \"cells\": [ "
+                        + "    { \"value\": 2.0 } "
+                        + "  ]"
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithSingleCellWithoutValue() {
+        assertTensorField("{{x:a}:0.0}",
+                createPutWithTensor("{ "
+                        + "  \"cells\": [ "
+                        + "    { \"address\": { \"x\": \"a\" } } "
+                        + "  ]"
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithDimensionsAndCells() {
+        assertTensorField("( {{z:-}:1.0} * {{x:a,y:b}:2.0,{x:c}:3.0} )",
+                createPutWithTensor("{ "
+                        + "  \"dimensions\": [\"x\",\"y\",\"z\"], "
+                        + "  \"cells\": [ "
+                        + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+                        + "      \"value\": 2.0 }, "
+                        + "    { \"address\": { \"x\": \"c\" }, "
+                        + "      \"value\": 3.0 } "
+                        + "  ]"
+                        + "}"));
+    }
+
+    @Test
+    public void testParsingOfTensorWithDimensionsAndCellsInDifferentJsonOrder() {
+        assertTensorField("( {{z:-}:1.0} * {{x:a,y:b}:2.0,{x:c}:3.0} )",
+                createPutWithTensor("{ "
+                        + "  \"cells\": [ "
+                        + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+                        + "      \"value\": 2.0 }, "
+                        + "    { \"address\": { \"x\": \"c\" }, "
+                        + "      \"value\": 3.0 } "
+                        + "  ],"
+                        + "  \"dimensions\": [\"x\",\"y\",\"z\"] "
+                        + "}"));
+    }
+
+    @Test
+    public void require_that_parser_propagates_datatype_parser_errors_predicate() {
+        assertParserErrorMatches(
+                "Error in document 'id:unittest:testpredicate::0' - could not parse field 'boolean' of type 'predicate': " +
+                        "line 1:10 no viable alternative at character '>'",
+
+                "[",
+                "      {",
+                "          'fields': {",
+                "              'boolean': 'timestamp > 9000'",
+                "          },",
+                "          'put': 'id:unittest:testpredicate::0'",
+                "      }",
+                "]"
+        );
+    }
+
+    @Test
+    public void require_that_parser_propagates_datatype_parser_errors_string_as_int() {
+        assertParserErrorMatches(
+                "Error in document 'id:unittest:testint::0' - could not parse field 'integerfield' of type 'int': " +
+                        "For input string: \" 1\"",
+
+                "[",
+                "      {",
+                "          'fields': {",
+                "              'integerfield': ' 1'",
+                "          },",
+                "          'put': 'id:unittest:testint::0'",
+                "      }",
+                "]"
+        );
+    }
+
+    @Test
+    public void require_that_parser_propagates_datatype_parser_errors_overflowing_int() {
+        assertParserErrorMatches(
+                "Error in document 'id:unittest:testint::0' - could not parse field 'integerfield' of type 'int': " +
+                        "For input string: \"281474976710656\"",
+
+                "[",
+                "      {",
+                "          'fields': {",
+                "              'integerfield': 281474976710656",
+                "          },",
+                "          'put': 'id:unittest:testint::0'",
+                "      }",
+                "]"
+        );
+    }
+
+    @Test
+    public void requireThatUpdatesForTensorFieldsAreNotSupported() {
+        try {
+            InputStream rawDoc = new ByteArrayInputStream(
+                    Utf8.toBytes("[ { \"update\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"tensorfield\": {"
+                            + "\"assign\": {} } } } ]"));
+            new JsonReader(types, rawDoc, parserFactory).next();
+            assertTrue("Exception not thrown", false);
+        } catch (IllegalArgumentException e) {
+            assertEquals("Updates to fields of type TENSOR is not yet supported (id='id:unittest:testtensor::0', field='tensorfield')",
+                    e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatUnknownDocTypeThrowsIllegalArgumentException() {
+        exception.expect(IllegalArgumentException.class);
+        exception.expectMessage(new Contains("Document type walrus does not exist"));
+
+        final String jsonData = inputJson(
+                "[",
+                "      {",
+                "          'put': 'id:ns:walrus::walrus1',",
+                "          'fields': {",
+                "              'aField': 42",
+                "          }",
+                "      }",
+                "]");
+
+        new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next();
+    }
+
+    private static final String TENSOR_DOC_ID = "id:unittest:testtensor::0";
+
+    private DocumentPut createPutWithoutTensor() {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("[ { \"put\": \"" + TENSOR_DOC_ID + "\", \"fields\": { } } ]"));
+        JsonReader reader = new JsonReader(types, rawDoc, parserFactory);
+        return (DocumentPut) reader.next();
+    }
+
+    private DocumentPut createPutWithTensor(String inputTensor) {
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("["
+                        + "  { \"put\": \"" + TENSOR_DOC_ID + "\", \"fields\": { \"tensorfield\": "
+                        + inputTensor
+                        + "  }}"
+                        + "]"));
+        JsonReader reader = new JsonReader(types, rawDoc, parserFactory);
+        return (DocumentPut) reader.next();
+    }
+
+    private static void assertTensorField(String expectedTensor, DocumentPut put) {
+        final Document doc = put.getDocument();
+        assertEquals("testtensor", doc.getId().getDocType());
+        assertEquals(TENSOR_DOC_ID, doc.getId().toString());
+        TensorFieldValue fieldValue = (TensorFieldValue)doc.getFieldValue(doc.getField("tensorfield"));
+        assertEquals(MapTensor.from(expectedTensor), fieldValue.getTensor().get());
+    }
+
+    // NOTE: Do not call this method multiple times from a test method as it's using the ExpectedException rule
+    private void assertParserErrorMatches(String expectedError, String... json) {
+        exception.expect(JsonReaderException.class);
+        exception.expectMessage(new Contains(expectedError));
+        String jsonData = inputJson(json);
+        new JsonReader(types, jsonToInputStream(jsonData), parserFactory).next();
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
new file mode 100644
index 00000000000..2e8354e3c6a
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
@@ -0,0 +1,384 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.json;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.DocumentOperation;
+import com.yahoo.document.DocumentPut;
+import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.Field;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import org.apache.commons.codec.binary.Base64;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.io.JsonStringEncoder;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.text.Utf8;
+
+/**
+ * Functional tests for com.yahoo.document.json.JsonWriter.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class JsonWriterTestCase {
+
+    private static final JsonFactory parserFactory = new JsonFactory();
+    private DocumentTypeManager types;
+
+    @Before
+    public void setUp() throws Exception {
+        types = new DocumentTypeManager();
+        {
+            DocumentType x = new DocumentType("smoke");
+            x.addField(new Field("something", DataType.STRING));
+            x.addField(new Field("nalle", DataType.STRING));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("mirrors");
+            StructDataType woo = new StructDataType("woo");
+            woo.addField(new Field("sandra", DataType.STRING));
+            woo.addField(new Field("cloud", DataType.STRING));
+            x.addField(new Field("skuggsjaa", woo));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testarray");
+            DataType d = new ArrayDataType(DataType.STRING);
+            x.addField(new Field("actualarray", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testset");
+            DataType d = new WeightedSetDataType(DataType.STRING, true, true);
+            x.addField(new Field("actualset", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testmap");
+            DataType d = new MapDataType(DataType.STRING, DataType.STRING);
+            x.addField(new Field("actualmap", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testraw");
+            DataType d = DataType.RAW;
+            x.addField(new Field("actualraw", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testpredicate");
+            DataType d = DataType.PREDICATE;
+            x.addField(new Field("actualpredicate", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testMapStringToArrayOfInt");
+            DataType value = new ArrayDataType(DataType.INT);
+            DataType d = new MapDataType(DataType.STRING, value);
+            x.addField(new Field("actualMapStringToArrayOfInt", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testsinglepos");
+            DataType d = PositionDataType.INSTANCE;
+            x.addField(new Field("singlepos", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testmultipos");
+            DataType d = new ArrayDataType(PositionDataType.INSTANCE);
+            x.addField(new Field("multipos", d));
+            types.registerDocumentType(x);
+        }
+        {
+            DocumentType x = new DocumentType("testtensor");
+            x.addField(new Field("tensorfield", DataType.TENSOR));
+            types.registerDocumentType(x);
+        }
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        types = null;
+    }
+
+    @Test
+    public final void smokeTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        roundTripEquality("id:unittest:smoke::whee", "{"
+                + " \"something\": \"smoketest\"," + " \"nalle\": \"bamse\""
+                + "}");
+    }
+
+    @Test
+    public final void hideEmptyStringsTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        final String fields = "{"
+                        + " \"something\": \"\"," + " \"nalle\": \"bamse\""
+                        + "}";
+        final String filteredFields = "{"
+                + " \"nalle\": \"bamse\""
+                + "}";
+
+        Document doc = readDocumentFromJson("id:unittest:smoke::whee", fields);
+        assertEqualJson(asDocument("id:unittest:smoke::whee", filteredFields), JsonWriter.toByteArray(doc));
+    }
+
+    private void roundTripEquality(final String docId, final String fields)
+            throws JsonParseException, JsonMappingException, IOException {
+        Document doc = readDocumentFromJson(docId, fields);
+        assertEqualJson(asDocument(docId, fields), JsonWriter.toByteArray(doc));
+    }
+
+    @Test
+    public final void structTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        roundTripEquality("id:unittest:mirrors::whee", "{ "
+                + "\"skuggsjaa\": {" + "\"sandra\": \"person\","
+                + " \"cloud\": \"another person\"}}");
+    }
+
+    @Test
+    public final void singlePosTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        roundTripEquality("id:unittest:testsinglepos::bamf", "{ \"singlepos\": \"N60.222333;E10.12\" }");
+    }
+
+    @Test
+    public final void multiPosTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        roundTripEquality("id:unittest:testmultipos::bamf", "{ \"multipos\": [ \"N0.0;E0.0\", \"S1.1;W1.1\", \"N10.2;W122.2\" ] }");
+    }
+
+    @Test
+    public final void arrayTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        roundTripEquality("id:unittest:testarray::whee", "{ \"actualarray\": ["
+                + " \"nalle\"," + " \"tralle\"]}");
+    }
+
+    @Test
+    public final void weightedSetTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        roundTripEquality("id:unittest:testset::whee", "{ \"actualset\": {"
+                + " \"nalle\": 2," + " \"tralle\": 7 }}");
+    }
+
+    @Test
+    public final void mapTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        final String fields = "{ \"actualmap\": ["
+                        + " { \"key\": \"nalle\", \"value\": \"kalle\"},"
+                        + " { \"key\": \"tralle\", \"value\": \"skalle\"} ]}";
+        final String docId = "id:unittest:testmap::whee";
+        Document doc = readDocumentFromJson(docId, fields);
+        // we have to do everything by hand to check, as maps are unordered, but
+        // are serialized as an ordered structure
+
+        ObjectMapper m = new ObjectMapper();
+        Map<?, ?> generated = m.readValue(JsonWriter.toByteArray(doc), Map.class);
+        assertEquals(docId, generated.get("id"));
+        // and from here on down there will be lots of unchecked casting and
+        // other fun. This is OK here, because if the casts fail, the should and
+        // will fail anyway
+        List<?> inputMap = (List<?>) m.readValue(Utf8.toBytes(fields), Map.class).get("actualmap");
+        List<?> generatedMap = (List<?>) ((Map<?, ?>) generated.get("fields")).get("actualmap");
+        assertEquals(populateMap(inputMap), populateMap(generatedMap));
+    }
+
+    // should very much blow up if the assumptions are incorrect
+    @SuppressWarnings("rawtypes")
+    private Map<Object, Object> populateMap(List<?> actualMap) {
+        Map<Object, Object> m = new HashMap<>();
+        for (Object o : actualMap) {
+            Object key = ((Map) o).get(JsonReader.MAP_KEY);
+            Object value = ((Map) o).get(JsonReader.MAP_VALUE);
+            m.put(key, value);
+        }
+        return m;
+    }
+
+    @Test
+    public final void rawTest() throws JsonParseException, JsonMappingException, IOException {
+        String payload = new String(
+                new JsonStringEncoder().quoteAsString(new Base64()
+                        .encodeToString(Utf8.toBytes("smoketest"))));
+        String docId = "id:unittest:testraw::whee";
+
+        String fields = "{ \"actualraw\": \"" + payload + "\"" + " }";
+        roundTripEquality(docId, fields);
+    }
+
+    @Test
+    public final void predicateTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        roundTripEquality("id:unittest:testpredicate::whee", "{ "
+                + "\"actualpredicate\": \"foo in [bar]\" }");
+    }
+
+    @Test
+    public final void stringToArrayOfIntMapTest() throws JsonParseException,
+            JsonMappingException, IOException {
+        String docId = "id:unittest:testMapStringToArrayOfInt::whee";
+        String fields = "{ \"actualMapStringToArrayOfInt\": ["
+                + "{ \"key\": \"bamse\", \"value\": [1, 2, 3] }" + "]}";
+        Document doc = readDocumentFromJson(docId, fields);
+        // we have to do everything by hand to check, as maps are unordered, but
+        // are serialized as an ordered structure
+
+        ObjectMapper m = new ObjectMapper();
+        Map<?, ?> generated = m.readValue(JsonWriter.toByteArray(doc), Map.class);
+        assertEquals(docId, generated.get("id"));
+        // and from here on down there will be lots of unchecked casting and
+        // other fun. This is OK here, because if the casts fail, the should and
+        // will fail anyway
+        List<?> inputMap = (List<?>) m.readValue(Utf8.toBytes(fields), Map.class).get("actualMapStringToArrayOfInt");
+        List<?> generatedMap = (List<?>) ((Map<?, ?>) generated.get("fields")).get("actualMapStringToArrayOfInt");
+        assertEquals(populateMap(inputMap), populateMap(generatedMap));
+    }
+
+    private Document readDocumentFromJson(final String docId,
+            final String fields) {
+        InputStream rawDoc = new ByteArrayInputStream(asFeed(
+                docId, fields));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        JsonReader.DocumentParseInfo raw = r.parseDocument().get();
+        DocumentType docType = r.readDocumentType(raw.documentId);
+        DocumentPut put = new DocumentPut(new Document(docType, raw.documentId));
+        r.readPut(put);
+        return put.getDocument();
+    }
+
+    private void assertEqualJson(byte[] expected, byte[] generated)
+            throws JsonParseException, JsonMappingException, IOException {
+        ObjectMapper m = new ObjectMapper();
+        Map<?, ?> exp = m.readValue(expected, Map.class);
+        Map<?, ?> gen = m.readValue(generated, Map.class);
+        assertEquals(exp, gen);
+    }
+
+    private byte[] asFeed(String docId, String fields) {
+        return completeDocString("put", docId, fields);
+    }
+
+    private byte[] asDocument(String docId, String fields) {
+        return completeDocString("id", docId, fields);
+    }
+
+    private byte[] completeDocString(String operation, String docId, String fields) {
+        return Utf8.toBytes("{\"" + operation + "\": \"" + docId + "\", \"fields\": " + fields + "}");
+    }
+
+    @Test
+    public void removeTest() {
+        final DocumentId documentId = new DocumentId("id:unittest:smoke::whee");
+        InputStream rawDoc = new ByteArrayInputStream(
+                Utf8.toBytes("["
+                + Utf8.toString(JsonWriter.documentRemove(documentId))
+                + "]"));
+        JsonReader r = new JsonReader(types, rawDoc, parserFactory);
+        DocumentOperation actualRemoveAsBaseType = r.next();
+        assertSame(DocumentRemove.class, actualRemoveAsBaseType.getClass());
+        assertEquals(actualRemoveAsBaseType.getId(), documentId);
+    }
+
+    @Test
+    public void testWritingWithoutTensorFieldValue() throws IOException {
+        roundTripEquality("id:unittest:testtensor::0", "{}");
+    }
+
+    @Test
+    public void testWritingOfEmptyTensor() throws IOException {
+        assertTensorRoundTripEquality("{}",
+                "{ \"dimensions\": [], \"cells\": [] }");
+    }
+
+    @Test
+    public void testWritingOfTensorWithCellsOnly() throws IOException {
+        assertTensorRoundTripEquality("{ "
+                + "  \"cells\": [ "
+                + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+                + "      \"value\": 2.0 }, "
+                + "    { \"address\": { \"x\": \"c\" }, "
+                + "      \"value\": 3.0 } "
+                + "  ]"
+                + "}", "{ "
+                + "  \"dimensions\": [\"x\", \"y\"], "
+                + "  \"cells\": [ "
+                + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+                + "      \"value\": 2.0 }, "
+                + "    { \"address\": { \"x\": \"c\" }, "
+                + "      \"value\": 3.0 } "
+                + "  ]"
+                + "}");
+    }
+
+    @Test
+    public void testWritingOfTensorWithSingleCellWithEmptyAddress() throws IOException {
+        assertTensorRoundTripEquality("{ "
+                + "  \"dimensions\": [], "
+                + "  \"cells\": [ "
+                + "    { \"address\": {}, \"value\": 2.0 } "
+                + "  ]"
+                + "}");
+    }
+
+    @Test
+    public void testWritingOfTensorWithDimensionsAndCells() throws IOException {
+        assertTensorRoundTripEquality("{ "
+                + "  \"dimensions\": [\"x\",\"y\",\"z\"], "
+                + "  \"cells\": [ "
+                + "    { \"address\": { \"x\": \"a\", \"y\": \"b\" }, "
+                + "      \"value\": 2.0 }, "
+                + "    { \"address\": { \"x\": \"c\" }, "
+                + "      \"value\": 3.0 } "
+                + "  ]"
+                + "}");
+    }
+
+    @Test
+    public void testWritingOfTensorFieldValueWithoutTensor() throws IOException {
+        DocumentType tensorType = types.getDocumentType("testtensor");
+        String docId = "id:unittest:testtensor::0";
+        Document doc = new Document(tensorType, docId);
+        doc.setFieldValue(tensorType.getField("tensorfield"), new TensorFieldValue());
+        assertEqualJson(asDocument(docId, "{ \"tensorfield\": {} }"), JsonWriter.toByteArray(doc));
+    }
+
+    private void assertTensorRoundTripEquality(String tensorField) throws IOException {
+        assertTensorRoundTripEquality(tensorField, tensorField);
+    }
+
+    private void assertTensorRoundTripEquality(String inputTensorField, String outputTensorField) throws IOException {
+        String inputFields = "{ \"tensorfield\": " + inputTensorField + " }";
+        String outputFields = "{ \"tensorfield\": " + outputTensorField + " }";
+        String docId = "id:unittest:testtensor::0";
+        Document doc = readDocumentFromJson(docId, inputFields);
+        assertEqualJson(asDocument(docId, outputFields), JsonWriter.toByteArray(doc));
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/outerdoc.sd b/document/src/test/java/com/yahoo/document/outerdoc.sd
new file mode 100644
index 00000000000..dbdf3c32cd5
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/outerdoc.sd
@@ -0,0 +1,6 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search outerdoc {
+  document outerdoc {
+    field innerdocuments type array<docindoc> { body }
+  }
+}
diff --git a/document/src/test/java/com/yahoo/document/select/BucketSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/BucketSelectorTestCase.java
new file mode 100644
index 00000000000..80282255dc5
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/select/BucketSelectorTestCase.java
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSelector;
+
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Date: Sep 6, 2007
+ *
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class BucketSelectorTestCase extends junit.framework.TestCase {
+
+    public BucketSelectorTestCase(String name) {
+        super(name);
+    }
+
+    public void testExpressions() throws Exception {
+        assertBucketCount("id = \"userdoc:ns:123:foobar\"", 1);
+        assertBucketCount("id = \"userdoc:ns:123:foo*\"", 0);
+        assertBucketCount("id == \"userdoc:ns:123:f?oo*\"", 1);
+        assertBucketCount("id =~ \"userdoc:ns:123:foo*\"", 0);
+        assertBucketCount("id =~ \"userdoc:ns:123:foo?\"", 0);
+        assertBucketCount("id.user = 123", 1);
+        assertBucketCount("id.user == 123", 1);
+        assertBucketCount("id.group = \"yahoo.com\"", 1);
+        assertBucketCount("id.group = \"yahoo.com\" or id.user=123", 2);
+        assertBucketCount("id.group = \"yahoo.com\" and id.user=123", 0);
+        assertBucketCount("id.group = \"yahoo.com\" and testdoctype1.hstringval=\"Doe\"", 1);
+        assertBucketCount("not id.group = \"yahoo.com\"", 0);
+        assertBucketCount("id.group != \"yahoo.com\"", 0);
+        assertBucketCount("id.group <= \"yahoo.com\"", 0);
+
+        assertBucketCount("id.bucket = 0x4000000000003018", 1); // Bucket 16:12312
+        assertBucketCount("id.bucket == 0x4000000000000258", 1); // Bucket 16:600
+
+        assertBucketCount("searchcolumn.3 == 1", 21845);
+
+        // Check that the correct buckets is found
+        assertBucket("id.bucket = 0x4000000000003018", new BucketId(16, 12312));
+        assertBucket("id.bucket == 0x4000000000000258", new BucketId(16, 600));
+
+        assertBucket("id = \"userdoc:ns:123:foobar\"", new BucketId(0xeafff5320000007bl));
+        assertBucket("id.user = 123", new BucketId(32, 123));
+        assertBucket("id.group = \"yahoo.com\"", new BucketId(32, 0x035837189a1acd50l));
+
+        // Check that overlapping works
+        Set<BucketId> expected = new TreeSet<BucketId>();
+        expected.add(new BucketId(32, 123));
+        expected.add(new BucketId(16, 123));
+        assertBuckets("id.user == 123 or id.bucket == 0x400000000000007b", expected);
+    }
+
+    public void assertBucketCount(String expr, int count) throws Exception {
+        BucketIdFactory factory = new BucketIdFactory();
+        BucketSelector selector = new BucketSelector(factory);
+        Set<BucketId> buckets = selector.getBucketList(expr);
+        assertEquals(count, buckets == null ? 0 : buckets.size());
+    }
+
+    public void assertBucket(String expr, BucketId bucket) throws Exception {
+        BucketIdFactory factory = new BucketIdFactory();
+        BucketSelector selector = new BucketSelector(factory);
+        Set<BucketId> buckets = selector.getBucketList(expr);
+        assertEquals(1, buckets == null ? 0 : buckets.size());
+        assertEquals(bucket, buckets.toArray()[0]);
+    }
+
+    public void assertBuckets(String expr, Set<BucketId> expected) throws Exception {
+        BucketIdFactory factory = new BucketIdFactory();
+        BucketSelector selector = new BucketSelector(factory);
+        Set<BucketId> actual = new TreeSet<BucketId>(selector.getBucketList(expr));
+        assertEquals(expected, actual);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
new file mode 100644
index 00000000000..1677592eecf
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
@@ -0,0 +1,782 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.select.convert.SelectionExpressionConverter;
+import com.yahoo.document.select.parser.ParseException;
+import com.yahoo.document.select.parser.TokenMgrError;
+import com.yahoo.yolean.Exceptions;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ * @author bratseth
+ */
+public class DocumentSelectorTestCase extends junit.framework.TestCase {
+
+    private static DocumentTypeManager manager = new DocumentTypeManager();
+
+    public DocumentSelectorTestCase(String name) {
+        super(name);
+    }
+
+    public void setUp() {
+        DocumentType type = new DocumentType("test");
+        type.addHeaderField("hint", DataType.INT);
+        type.addHeaderField("hfloat", DataType.FLOAT);
+        type.addHeaderField("hstring", DataType.STRING);
+        type.addField("content", DataType.STRING);
+
+        StructDataType mystruct = new StructDataType("mystruct");
+        mystruct.addField(new Field("key", DataType.INT, false));
+        mystruct.addField(new Field("value", DataType.STRING, false));
+        type.addHeaderField("mystruct", mystruct);
+
+        ArrayDataType structarray = new ArrayDataType(mystruct);
+        type.addField("structarray", structarray);
+
+        type.addField("stringweightedset", new WeightedSetDataType(DataType.STRING, false, false));
+        type.addField("mymap", new MapDataType(DataType.INT, DataType.STRING));
+        type.addField("structarrmap", new MapDataType(DataType.STRING, structarray));
+
+        ArrayDataType intarray = new ArrayDataType(DataType.INT);
+        type.addField("intarray", intarray);
+
+        manager.registerDocumentType(type);
+
+        // Create strange doctypes using identifiers within them, which we
+        // can use to verify they are still parsed as doctypes.
+        manager.registerDocumentType(new DocumentType("notandor"));
+        manager.registerDocumentType(new DocumentType("ornotand"));
+        manager.registerDocumentType(new DocumentType("andornot"));
+        manager.registerDocumentType(new DocumentType("idid"));
+        manager.registerDocumentType(new DocumentType("usergroup"));
+    }
+
+    public void testParsing() throws ParseException {
+        assertParse("3.14 > 0");
+        assertParse("-999 > 0");
+        assertParse("150000.0 > 0", "15e4 > 0");
+        assertParse("3.4E-4 > 0", "3.4e-4 > 0");
+        assertParse("\" Test \" = \"*\"");
+        assertParse("id = \"*\"", "id = '*'");
+        assertParse("id.group == 3");
+        assertParse("id.namespace = \"*\"");
+        assertParse("id.hash() > 0");
+        assertParse("id.namespace.hash() > 0");
+        assertParse("id.order(5,2) > 100");
+        assertParse("searchcolumn.10 = 6");
+        assertParse("music.artist = \"*\"");
+        assertParse("music.artist.lowercase() = \"*\"");
+        assertParse("music_.artist = \"*\"");
+        assertParse("music_foo.artist = \"*\"");
+        assertParse("music_foo_.artist = \"*\"");
+        assertParse("(4 + 3) > 0", "(4+3) > 0");
+        assertParse("1 + 1 > 0", "1 +1 > 0");
+        assertParse("1 + -1 > 0", "1 + -1 > 0");
+        assertParse("1 + 1.0 > 0", "1 + +1.0 > 0");
+        assertParse("1 - 1 > 0", "1 -1 > 0");
+        assertParse("1 - -1 > 0", "1 - -1 > 0");
+        assertParse("1 - 1.0 > 0", "1 - +1.0 > 0");
+        assertParse("1 + 2 * 3 - 10 % 2 / 3 > 0", "1   +2 *  3- 10%2   /3  > 0");
+        assertParse("((43 + 14) / 34) > 0");
+        assertParse("(34 * ((3 - 1) % 4)) > 0");
+        assertParse("true");
+        assertParse("false");
+        assertParse("music");
+        assertParse("(music or book)");
+        assertParse("music or book", "music or book");
+        assertParse("(music or (book and video))");
+        assertParse("music or (book and video)", "music or (book and video)");
+        assertParse("((music or book) and video)");
+        assertParse("(music or book) and video", "(music or book) and video");
+        assertParse("music.test > 0");
+        assertParse("music.artist = \"*john*\"");
+        assertParse("music.length >= 180");
+        assertParse("true or not false and true", "true oR nOt false And true");
+        assertParse("(true or false) and true", "(true oR false) aNd true");
+        assertParse("music.expire > now()");
+        assertParse("music.expire > now() - 300");
+        assertParse("now or now_search");
+        assertParse("(music.expire / 1000) > (now() - 300)");
+    }
+
+    public void testReservedWords() throws ParseException {
+        assertParse(null, "id == 'id' or id_t or idtype"); // ignore canonical form
+        assertParse(null, "searchcolumn == 1 or searchcolumn_t or searchcolumntype");
+        assertParse(null, "id.scheme == 'scheme' or scheme_t or schemetype");
+        assertParse(null, "id.namespace == 'namespace' or namespace_t or namespacetype");
+        assertParse(null, "id.specific == 'specific' or specific_t or specifictype");
+        assertParse(null, "id.user == 'user' or user_t or usertype");
+        assertParse(null, "id.group == 'group' or group_t or grouptype");
+        assertParse(null, "id.bucket == 'bucket' or bucket_t or buckettype");
+        assertParse(null, "null == 'null' or null_t or nulltype");
+        assertParse(null, "true or true_t or truetype");
+        assertParse(null, "false or false_t or falsetype");
+        assertParse(null, "true or and_t or andtype");
+        assertParse(null, "true or or_t or ortype");
+    }
+
+    public void testCjkParsing() throws ParseException {
+        assertParse("music.artist = \"\\u4f73\\u80fd\\u7d22\\u5c3c\\u60e0\\u666e\"",
+                    "music.artist = \"\u4f73\u80fd\u7d22\u5c3c\u60e0\u666e\"");
+        assertParse("music.artist = \"\\u4f73\\u80fd\\u7d22\\u5c3c\\u60e0\\u666e\"",
+                    "music.artist = \"\\u4f73\\u80fd\\u7d22\\u5c3c\\u60e0\\u666e\"");
+    }
+
+    public void testParseTerminals() throws ParseException {
+        // Test number values.
+        assertParse("true");
+        assertParse("music.hmm == 123");
+        assertParse("music.hmm == 123.53", "music.hmm == +123.53");
+        assertParse("music.hmm == -123.5");
+        assertParse("music.hmm == 2.3412352E8", "music.hmm == 234123.52e3");
+        assertParse("music.hmm == -234.12352", "music.hmm == -234123.52E-3");
+        assertParse("music.hmm < aaa");
+
+        // Test string values.
+        assertParse("music.hmm == \"test\"");
+
+        // Test map and struct stuff.
+        assertParse("music.hmm{test} == \"test\"");
+        assertParse("music.hmm{test}.foo[3].key == \"test\"");
+
+        // Test whitespaces.
+        assertParse("music.hmm == \"te st \"");
+        assertParse("music.hmm == \"test\"", " \t music.hmm\t==  \t \"test\"\t");
+
+        // Test escaping.
+        assertParse("music.hmm == \"tab\\ttest\"");
+        assertParse("music.hmm == \"tab\\u0666test\"", "music.hmm == \"tab\\u0666test\"");
+        assertParse("music.hmm == \"tabcomplete\"", "music.hmm == \"tabcomplete\"");
+        assertParse("music.hmm == \"tabysf\"", "music.hmm == \"tab\\ysf\""); // excessive escapes are removed
+        assertParse("music.h == \"\\ttx48 \\n\"", "music.h == \"\\tt\\x48 \\n\"");
+
+        // Test illegal operator.
+        assertParseError("music.hmm <> 12", "Exception parsing document selector 'music.hmm <> 12': Encountered \" \">\" \"> \"\" at line 1, column 12.");
+
+        // Test comparison operators.
+        assertParse("music.hmm >= 123");
+        assertParse("music.hmm > 123");
+        assertParse("music.hmm <= 123");
+        assertParse("music.hmm < 123");
+        assertParse("music.hmm != 123");
+
+        // Test defined.
+        assertParse("music.hmm");
+
+        // Test boolean expressions.
+        assertParse("true", "TRUE");
+        assertParse("false", "FALSE");
+        assertParse("true", "true");
+        assertParse("false", "false");
+        assertParse("false", "faLSe");
+
+        // Test document types.
+        assertParse("mytype");
+
+        // Test document id.
+        assertParse("id == \"userdoc:ns:mytest\"");
+        assertParse("id.namespace == \"myspace\"");
+        assertParse("id.scheme == \"userdoc\"");
+        assertParse("id.type == \"mytype\"");
+        assertParse("id.user == 1234");
+        assertParse("id.bucket == 8388608", "id.bucket == 0x800000");
+        assertParse("id.bucket == 8429568", "id.bucket == 0x80a000");
+        assertParse("id.bucket == -9223372036854775566",
+                    "id.bucket == 0x80000000000000f2");
+        assertParse("id.group == \"yahoo.com\"");
+        assertParse("id.specific == \"mypart\"");
+
+        // Test search column stuff.
+        assertParse("searchcolumn.10 = 6");
+
+        // Test other operators.
+        assertParse("id.scheme = \"*doc\"");
+        assertParse("music.artist =~ \"(john|barry|shrek)\"");
+
+        // Verify functions.
+        assertParse("id.hash() == 124");
+        assertParse("id.specific.hash() == 124");
+        assertParse("music.artist.lowercase() == \"chang\"");
+        assertParse("music.artist.lowercase().hash() == 124");
+        assertParse("music.version() == 8");
+        assertParse("music == 8"); // will evaluate to false
+
+        // Value grouping.
+        assertParse("(123) < (200)", "(123) < (200)");
+        assertParse("(\"hmm\") < (id.scheme)", "(\"hmm\") < (id.scheme)");
+
+        // Arithmetics.
+        assertParse("(1 + 2) > 1");
+        assertParse("1 + 2 > 1", "1 + 2 > 1");
+        assertParse("(1 - 2) > 1");
+        assertParse("(1 * 2) > 1");
+        assertParse("(1 / 2) > 1");
+        assertParse("(1 % 2) > 1");
+        assertParse("((1 + 2) * (4 - 2)) == 1");
+        assertParse("(1 + 2) * (4 - 2) == 1", "(1 + 2) * (4 - 2) == 1");
+        assertParse("((23 + 643) / (34 % 10)) > 34");
+        assertParse("23 + 643 / 34 % 10 > 34", "23 + 643 / 34 % 10 > 34");
+    }
+
+    public void testParseReservedTokens() throws ParseException {
+        assertParse("user.fieldName == \"fieldValue\""); // reserved word as document type name
+        assertParse("documentName.user == \"fieldValue\""); // reserved word as field type name
+        assertParse("group.fieldName == \"fieldValue\""); // reserved word as document type name
+        assertParse("documentName.group == \"fieldValue\""); // reserved word as field type name
+    }
+
+    public void testParseBranches() throws ParseException {
+        assertParse("((true or false) and (false or true))");
+        assertParse("(true or (not false and not true))");
+        assertParse("((243) < 300 and (\"FOO\").lowercase() == (\"foo\"))");
+    }
+
+    public void testDocumentUpdate() throws ParseException {
+        DocumentUpdate upd = new DocumentUpdate(manager.getDocumentType("test"), new DocumentId("doc:myspace:anything"));
+        assertEquals(Result.TRUE, evaluate("test", upd));
+        assertEquals(Result.FALSE, evaluate("music", upd));
+        assertEquals(Result.TRUE, evaluate("test or music", upd));
+        assertEquals(Result.FALSE, evaluate("test and music", upd));
+        assertEquals(Result.INVALID, evaluate("test.hint", upd));
+        assertEquals(Result.INVALID, evaluate("test.anything", upd));
+        assertEquals(Result.INVALID, evaluate("test.hint < 24", upd));
+        // TODO Fails: assertEquals(Result.TRUE, evaluate("test.hint + 1 > 13", upd));
+    }
+
+    public void testInvalidLogic() throws ParseException {
+        DocumentPut put = new DocumentPut(manager.getDocumentType("test"), new DocumentId("doc:scheme:"));
+        DocumentUpdate upd = new DocumentUpdate(manager.getDocumentType("test"), new DocumentId("doc:scheme:"));
+
+        assertEquals(Result.FALSE, evaluate("test.content", put)); // BROKEN
+        assertEquals(Result.INVALID, evaluate("test.content", upd));
+
+        assertEquals(Result.FALSE, evaluate("test.content = 1", put)); // BROKEN
+        assertEquals(Result.INVALID, evaluate("test.content = 1", upd));
+
+        assertEquals(Result.FALSE, evaluate("test.content = 1 and true", put)); // BROKEN
+        assertEquals(Result.INVALID, evaluate("test.content = 1 and true", upd));
+
+        assertEquals(Result.FALSE, evaluate("test.content = 1 or true", put)); // BROKEN
+        assertEquals(Result.TRUE, evaluate("test.content = 1 or true", upd));
+
+        assertEquals(Result.FALSE, evaluate("test.content = 1 and false",  put));
+        assertEquals(Result.FALSE, evaluate("test.content = 1 and false",  upd));
+
+        assertEquals(Result.FALSE, evaluate("test.content = 1 or false", put)); // BROKEN
+        assertEquals(Result.INVALID, evaluate("test.content = 1 or false", upd));
+
+        assertEquals(Result.FALSE, evaluate("true and test.content = 1",  put)); // BROKEN
+        assertEquals(Result.INVALID, evaluate("true and test.content = 1",  upd));
+
+        assertEquals(Result.FALSE, evaluate("true or test.content = 1", put)); // BROKEN
+        assertEquals(Result.TRUE, evaluate("true or test.content = 1", upd));
+
+        assertEquals(Result.FALSE, evaluate("false and test.content = 1",  put));
+        assertEquals(Result.FALSE, evaluate("false and test.content = 1",  upd));
+
+        assertEquals(Result.FALSE, evaluate("false or test.content = 1",  put)); // BROKEN
+        assertEquals(Result.INVALID, evaluate("false or test.content = 1",  upd));
+    }
+
+    List<DocumentPut> createDocs() {
+        List<DocumentPut> documents = new ArrayList<>();
+        documents.add(createDocument("doc:myspace:anything", 24, 2.0f, "foo", "bar"));
+        documents.add(createDocument("doc:anotherspace:foo", 13, 4.1f, "bar", "foo"));
+        documents.add(createDocument("userdoc:myspace:1234:mail1", 15, 1.0f, "some", "some"));
+        documents.add(createDocument("userdoc:myspace:5678:bar", 14, 2.4f, "Yet", "More"));
+        documents.add(createDocument("orderdoc(31,19):ns2:1234:5678:foo", 14, 2.4f, "Yet", "More"));
+        documents.add(createDocument("id:myspace:test:n=2345:mail2", 15, 1.0f, "bar", "baz"));
+        documents.add(createDocument("id:myspace:test:g=mygroup:qux", 15, 1.0f, "quux", "corge"));
+        documents.add(createDocument("doc:myspace:missingint", null, 2.0f, null, "bar"));
+
+        // Add some array/struct info to doc 1
+        Struct sval = new Struct(documents.get(1).getDocument().getField("mystruct").getDataType());
+        sval.setFieldValue("key", new IntegerFieldValue(14));
+        sval.setFieldValue("value", new StringFieldValue("structval"));
+        documents.get(1).getDocument().setFieldValue("mystruct", sval);
+        Array<Struct> aval = new Array<>(documents.get(1).getDocument().getField("structarray").getDataType());
+        {
+            Struct sval1 = new Struct(aval.getDataType().getNestedType());
+            sval1.setFieldValue("key", new IntegerFieldValue(15));
+            sval1.setFieldValue("value", new StringFieldValue("structval1"));
+            Struct sval2 = new Struct(aval.getDataType().getNestedType());
+            sval2.setFieldValue("key", new IntegerFieldValue(16));
+            sval2.setFieldValue("value", new StringFieldValue("structval2"));
+            aval.add(sval1);
+            aval.add(sval2);
+        }
+        documents.get(1).getDocument().setFieldValue("structarray", aval);
+
+        MapFieldValue<IntegerFieldValue, StringFieldValue> mval =
+                new MapFieldValue<>((MapDataType)documents.get(1).getDocument().getField("mymap")
+                                                                                             .getDataType());
+        mval.put(new IntegerFieldValue(3), new StringFieldValue("a"));
+        mval.put(new IntegerFieldValue(5), new StringFieldValue("b"));
+        mval.put(new IntegerFieldValue(7), new StringFieldValue("c"));
+        documents.get(1).getDocument().setFieldValue("mymap", mval);
+
+        MapFieldValue<StringFieldValue, Array> amval =
+                new MapFieldValue<>((MapDataType)documents.get(1).getDocument().getField("structarrmap")
+                                                                                 .getDataType());
+        amval.put(new StringFieldValue("foo"), aval);
+
+        Array<Struct> abval = new Array<>(documents.get(1).getDocument().getField("structarray").getDataType());
+        {
+            Struct sval1 = new Struct(aval.getDataType().getNestedType());
+            sval1.setFieldValue("key", new IntegerFieldValue(17));
+            sval1.setFieldValue("value", new StringFieldValue("structval3"));
+            Struct sval2 = new Struct(aval.getDataType().getNestedType());
+            sval2.setFieldValue("key", new IntegerFieldValue(18));
+            sval2.setFieldValue("value", new StringFieldValue("structval4"));
+            abval.add(sval1);
+            abval.add(sval2);
+        }
+
+        amval.put(new StringFieldValue("bar"), abval);
+        documents.get(1).getDocument().setFieldValue("structarrmap", amval);
+
+        WeightedSet<StringFieldValue> wsval = new WeightedSet<>(documents.get(1).getDocument().getField("stringweightedset")
+                                                                     .getDataType());
+        wsval.add(new StringFieldValue("foo"));
+        wsval.add(new StringFieldValue("val1"));
+        wsval.add(new StringFieldValue("val2"));
+        wsval.add(new StringFieldValue("val3"));
+        wsval.add(new StringFieldValue("val4"));
+        documents.get(1).getDocument().setFieldValue("stringweightedset", wsval);
+
+        // Add empty array/struct to doc 2
+        Struct sval3 = new Struct(documents.get(2).getDocument().getField("mystruct").getDataType());
+        documents.get(2).getDocument().setFieldValue("mystruct", sval3);
+        Array aval2 = new Array(documents.get(2).getDocument().getField("structarray").getDataType());
+        documents.get(2).getDocument().setFieldValue("structarray", aval2);
+
+        Array<IntegerFieldValue> intvals1 = new Array<>(documents.get(0).getDocument().getField("intarray")
+                                                                                  .getDataType());
+        intvals1.add(new IntegerFieldValue(12));
+        intvals1.add(new IntegerFieldValue(40));
+        intvals1.add(new IntegerFieldValue(60));
+        intvals1.add(new IntegerFieldValue(84));
+        documents.get(0).getDocument().setFieldValue("intarray", intvals1);
+
+        Array<IntegerFieldValue> intvals2 = new Array<>(documents.get(1).getDocument().getField("intarray")
+                                                                                  .getDataType());
+        intvals2.add(new IntegerFieldValue(3));
+        intvals2.add(new IntegerFieldValue(56));
+        intvals2.add(new IntegerFieldValue(23));
+        intvals2.add(new IntegerFieldValue(9));
+        documents.get(1).getDocument().setFieldValue("intarray", intvals2);
+
+        return documents;
+    }
+
+    public void testOperators() throws ParseException {
+        List<DocumentPut> documents = createDocs();
+
+        // Check that comparison operators work.
+        assertEquals(Result.TRUE, evaluate("", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("30 < 10", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("10 < 30", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("30 < 10", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("10 < 30", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("30 <= 10", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("10 <= 30", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 <= 30", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("10 >= 30", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 >= 10", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 >= 30", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("10 > 30", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 > 10", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("30 == 10", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 == 30", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 != 10", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("30 != 30", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"foo\" != \"bar\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("\"foo\" != \"foo\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("'foo' == \"bar\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("'foo' == \"foo\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("\"bar\" = \"a\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"bar\" = \"*a*\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("\"bar\" = \"\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"\" = \"\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("\"bar\" =~ \"^a$\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"bar\" =~ \"a\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"bar\" =~ \"\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"\" =~ \"\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("30 = 10", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 = 30", documents.get(0)));
+
+        // Mix of types should within numbers, but otherwise not match
+        assertEquals(Result.FALSE, evaluate("30 < 10.2", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("10.2 < 30", documents.get(0)));
+        assertEquals(Result.INVALID, evaluate("30 < \"foo\"", documents.get(0)));
+        assertEquals(Result.INVALID, evaluate("30 > \"foo\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("30 != \"foo\"", documents.get(0)));
+        assertEquals(Result.INVALID, evaluate("14.2 <= \"foo\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("null == null", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("null = null", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("\"bar\" == null", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("14.3 == null", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("null = 0", documents.get(0)));
+
+        // Field values
+        assertEquals(Result.TRUE, evaluate("test.hint = 24", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.hint = 24", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.hint = 13", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.hint = 13", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.hfloat = 2.0", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.hfloat = 1.0", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.hfloat = 4.1", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.hfloat > 4.09 and test.hfloat < 4.11", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.content = \"bar\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.content = \"bar\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.content = \"foo\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.content = \"foo\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.hstring == test.content", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.hstring == test.content", documents.get(2)));
+        assertEquals(Result.TRUE, evaluate("test.hint + 1 > 13", documents.get(1)));
+
+        // Document types.
+        assertEquals(Result.TRUE, evaluate("test", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("nonexisting", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("nonexisting.reallynonexisting", documents.get(0)));
+        assertEquals(Result.INVALID, evaluate("nonexisting.reallynonexisting > 13", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("true.foo", documents.get(0)));
+
+        // Field existence
+        assertEquals(Result.TRUE, evaluate("test.hint", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.hstring", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.hint", documents.get(7)));
+        assertEquals(Result.FALSE, evaluate("test.hstring", documents.get(7)));
+        assertEquals(Result.TRUE, evaluate("not test.hint", documents.get(7)));
+        assertEquals(Result.TRUE, evaluate("not test.hstring", documents.get(7)));
+
+        // Id values.
+        assertEquals(Result.TRUE, evaluate("id == \"doc:myspace:anything\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate(" iD==  \"doc:myspace:anything\"  ", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("id == \"doc:myspa:nything\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("Id.scHeme == \"doc\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("id.scheme == \"userdoc\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("id.type == \"test\"", documents.get(5)));
+        assertEquals(Result.FALSE, evaluate("id.type == \"wrong\"", documents.get(5)));
+        assertEquals(Result.TRUE, evaluate("Id.namespaCe == \"myspace\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("id.NaMespace == \"pace\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("id.specific == \"anything\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("id.user=1234", documents.get(2)));
+        assertEquals(Result.TRUE, evaluate("id.user=1234", documents.get(4)));
+        assertEquals(Result.TRUE, evaluate("id.group=\"1234\"", documents.get(4)));
+        assertEquals(Result.TRUE, evaluate("id.user=2345", documents.get(5)));
+        assertEquals(Result.TRUE, evaluate("id.group=\"mygroup\"", documents.get(6)));
+
+        assertEquals(Result.TRUE, evaluate("id.order(31,19)=5678", documents.get(4)));
+        assertEquals(Result.TRUE, evaluate("id.order(31,19)<=5678", documents.get(4)));
+        assertEquals(Result.FALSE, evaluate("id.order(31,19)>5678", documents.get(4)));
+        assertEquals(Result.FALSE, evaluate("id.order(25,23)==5678", documents.get(4)));
+        assertEquals(Result.FALSE, evaluate("id.order(31,19)=5678", documents.get(3)));
+
+        assertError("id.user == 1234", documents.get(0), "User identifier is null.");
+        assertError("id.group == 1234", documents.get(3), "Group identifier is null.");
+        assertError("id.group == \"yahoo\"", documents.get(3), "Group identifier is null.");
+        assertError("id.type == \"unknown\"", documents.get(0), "Document id doesn't have doc type.");
+
+        assertEquals(Result.TRUE, evaluate("searchcolumn.10 == 6", documents.get(3)));
+
+        // Branch operators.
+        assertEquals(Result.FALSE, evaluate("true and false", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("true and true", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("true or false", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("false or false", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("false and true or true and true", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("false or true and true or false", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("not false", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("not true", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("true and not false or false", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("((243 < 300) and (\"FOO\".lowercase() == \"foo\"))", documents.get(0)));
+
+        // Invalid branching. test.content = 1 is invalid.
+        assertEquals(Result.FALSE, evaluate("test.content = 1 and true", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.content = 1 or true", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.content = 1 and false", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.content = 1 or false", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("true and test.content = 1", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("true or test.content = 1", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("false and test.content = 1", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("false or test.content = 1", documents.get(0)));
+
+        // Functions.
+        assertEquals(Result.FALSE, evaluate("test.hstring.lowercase() == \"Yet\"", documents.get(3)));
+        assertEquals(Result.TRUE, evaluate("test.hstring.lowercase() == \"yet\"", documents.get(3)));
+        assertEquals(Result.FALSE, evaluate("test.hfloat.lowercase() == \"yet\"", documents.get(3)));
+        assertEquals(Result.TRUE, evaluate("\"bar\".hash() == -270124981", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"bar\".hash().abs() == 270124981", documents.get(0)));
+        assertError("null.hash() == 22460089", documents.get(0), "Can not invoke 'hash()' on 'null' because that term evaluated to null");
+        assertEquals(Result.FALSE, evaluate("(0.234).hash() == 123", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("(0.234).lowercase() == 123", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("\"foo\".hash() == 123", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("(234).hash() == 123", documents.get(0)));
+        // Arithmetics
+        assertEquals(Result.TRUE, evaluate("id.specific.hash() = 596580044", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("id.specific.hash() % 10 = 4", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("id.specific.hash() % 10 = 2", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("\"foo\" + \"bar\" = \"foobar\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("\"foo\" + 4 = 25", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("34.0 % 4 = 4", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("-6 % 10 = -6", documents.get(0)));
+
+        // Test now(). Assumes that now() is never 0
+        assertEquals(Result.FALSE, evaluate("0 > now()", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("0 < now()", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("0 < now() - 10", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("now() - 20 < now() - 10", documents.get(0)));
+        long secondsNow = System.currentTimeMillis() / 1000;
+        assertEquals(Result.TRUE, evaluate("now() - " + secondsNow + " < 2", documents.get(0)));
+
+        // Structs and arrays
+        // Commented out tests work in C++.. Don't want to start altering
+        // java code before talking to Simon.. Leaving it as is, as the
+        // needed functionality of testing for existance already works.
+        assertEquals(Result.FALSE, evaluate("test.mystruct", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.mystruct", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mystruct", documents.get(2)));
+        assertEquals(Result.TRUE, evaluate("test.mystruct == test.mystruct", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.mystruct == test.mystruct", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mystruct != test.mystruct", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.mystruct != test.mystruct", documents.get(1)));
+        //assertEquals(Result.INVALID, evaluate("test.mystruct < test.mystruct", documents.get(0)));
+        //assertEquals(Result.FALSE, evaluate("test.mystruct < test.mystruct", documents.get(1)));
+        //assertEquals(Result.INVALID, evaluate("test.mystruct < 5", documents.get(1)));
+        //assertEquals(Result.INVALID, evaluate("test.mystruct == \"foo\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarray", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.structarray", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarray", documents.get(2)));
+        assertEquals(Result.TRUE, evaluate("test.structarray == test.structarray", documents.get(0)));
+        //assertEquals(Result.INVALID, evaluate("test.structarray < test.structarray", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.structarray == test.structarray", documents.get(1)));
+        //assertEquals(Result.FALSE, evaluate("test.structarray < test.structarray", documents.get(1)));
+
+        assertEquals(Result.FALSE, evaluate("test.structarray.key == 15", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.structarray[4].key == 15", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.structarray", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.structarray.key == 15", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.structarray[1].key == 16", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.structarray[1].key = 16", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarray.value == \"structval1\"", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.structarray[4].value == \"structval1\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.structarray.value == \"structval1\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.structarray[0].value == \"structval1\"", documents.get(1)));
+        // Globbing of array-of-struct fields
+        assertEquals(Result.FALSE, evaluate("test.structarray.key = 15", documents.get(0))); // Fallback
+        assertEquals(Result.FALSE, evaluate("test.structarray.key = 15", documents.get(2))); // Fallback
+        assertEquals(Result.TRUE, evaluate("test.structarray.key = 15", documents.get(1))); // Fallback
+        assertEquals(Result.FALSE, evaluate("test.structarray.value = \"structval2\"", documents.get(2)));
+        assertEquals(Result.TRUE, evaluate("test.structarray.value = \"*ctval*\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.structarray[1].value = \"structval2\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarray[1].value = \"batman\"", documents.get(1)));
+        // Regexp of array-of-struct fields
+        assertEquals(Result.TRUE, evaluate("test.structarray.value =~ \"structval[1-9]\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarray.value =~ \"structval[a-z]\"", documents.get(1)));
+        // Globbing/regexp of struct fields
+        assertEquals(Result.FALSE, evaluate("test.mystruct.value = \"struc?val\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.mystruct.value = \"struc?val\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mystruct.value =~ \"struct.*\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.mystruct.value =~ \"struct.*\"", documents.get(1)));
+
+        assertEquals(Result.FALSE, evaluate("test.intarray < 5", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.intarray < 5", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.intarray > 80", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.intarray > 80", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.intarray >= 84", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.intarray <= 3", documents.get(1)));
+
+        // Interesting property ...
+        assertEquals(Result.TRUE, evaluate("test.intarray == 84", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.intarray != 84", documents.get(0)));
+
+        assertEquals(Result.TRUE, evaluate("test.structarray[$x].key == 15 AND test.structarray[$x].value == \"structval1\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarray[$x].key == 15 AND test.structarray[$x].value == \"structval2\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.structarray[$x].key == 15 AND test.structarray[$y].value == \"structval2\"", documents.get(1)));
+
+        assertEquals(Result.FALSE, evaluate("test.mymap", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.mymap", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.mymap{3}", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap{9}", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.mymap{3} == \"a\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap{3} == \"b\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap{9} == \"b\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap == \"a\"", documents.get(1))); // Keys only
+        assertEquals(Result.TRUE, evaluate("test.mymap{3} = \"a\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap{3} = \"b\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.mymap{3} =~ \"a\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap{3} =~ \"b\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.mymap.value = \"a\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap.value = \"d\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.mymap.value =~ \"a\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap.value =~ \"d\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.mymap == 3", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.mymap == 4", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.mymap = 3", documents.get(1))); // Fallback to ==
+        assertEquals(Result.FALSE, evaluate("test.mymap = 4", documents.get(1))); // Fallback to ==
+
+        assertEquals(Result.TRUE, evaluate("test.structarrmap{$x}[$y].key == 15 AND test.structarrmap{$x}[$y].value == \"structval1\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.structarrmap.value[$y].key == 15 AND test.structarrmap.value[$y].value == \"structval1\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarrmap{$x}[$y].key == 15 AND test.structarrmap{$x}[$y].value == \"structval2\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarrmap.value[$y].key == 15 AND test.structarrmap.value[$y].value == \"structval2\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarrmap{$x}[$y].key == 15 AND test.structarrmap{$y}[$x].value == \"structval2\"", documents.get(1)));
+
+        assertEquals(Result.TRUE, evaluate("test.stringweightedset", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.stringweightedset{val1}", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.stringweightedset{val1} == 1", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.stringweightedset{val1} == 2", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.stringweightedset == \"val1\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.stringweightedset = \"val*\"", documents.get(1)));
+        assertEquals(Result.TRUE, evaluate("test.stringweightedset =~ \"val[0-9]\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.stringweightedset == \"val5\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.stringweightedset = \"val5\"", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.stringweightedset =~ \"val5\"", documents.get(1)));
+
+        assertEquals(Result.TRUE, evaluate("test.structarrmap{$x}.key == 15 AND test.stringweightedset{$x}", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.structarrmap{$x}.key == 17 AND test.stringweightedset{$x}", documents.get(1)));
+    }
+
+    public void testTicket1769674() throws ParseException {
+        assertParseError("music.uri=\"junk",
+                         "Lexical error at line -1, column 17.  Encountered: <EOF> after : \"\\\"junk\"");
+    }
+
+    public void testThatVisitingReportsCorrectResult() throws ParseException {
+        assertVisitWithValidNowWorks("music.expire > now()");
+        assertVisitWithValidNowWorks("music.expire > now() and video.expire > now()");
+        assertVisitWithValidNowWorks("music.expire > now() or video");
+        assertVisitWithValidNowWorks("music.expire > now() or video.date < 300");
+        assertVisitWithValidNowWorks("video.date < 300 or music.expire > now()");
+        assertVisitWithValidNowWorks("video.date < 300 and music.expire > now()");
+        assertVisitWithValidNowWorks("music.insertdate > now() - 300 and video.expire > now() - 3600");
+
+        assertVisitWithoutNowWorks("test.structarrmap{$x}[$y].key == 15 AND test.structarrmap{$x}[$y].value == \"structval1\"");
+        assertVisitWithoutNowWorks("music.artist.lowercase() == \"chang\"");
+
+        assertVisitWithInvalidNowFails("music.expire > now() + 300", "Arithmetic operator '+' is not supported");
+        assertVisitWithInvalidNowFails("music.expire < now()", "Comparison operator '<' is not supported");
+        assertVisitWithInvalidNowFails("music.expire >= now()", "Comparison operator '>=' is not supported");
+        assertVisitWithInvalidNowFails("now() > now()", "Left hand side of comparison must be a document field");
+        assertVisitWithInvalidNowFails("music.name.hash() > now()", "Only attribute items are supported");
+    }
+
+    public void testThatSelectionIsConvertedToQueries() throws ParseException {
+        assertThatQueriesAreCreated("music.expire > now()", Arrays.asList("music"), Arrays.asList("expire:>now(0)"));
+        assertThatQueriesAreCreated("music.expire > now() - 300", Arrays.asList("music"), Arrays.asList("expire:>now(300)"));
+        assertThatQueriesAreCreated("music.expire > now() - 300 and video.expire > now() - 3600", Arrays.asList("music", "video"), Arrays.asList("expire:>now(300)", "expire:>now(3600)"));
+        assertThatQueriesAreCreated("music.expire > now() - 300 or video", Arrays.asList("music"), Arrays.asList("expire:>now(300)"));
+        assertVisitWithInvalidNowFails("music.field1 > now() - 300 and music.field2 > now() - 300", "Specifying multiple document types is not allowed");
+        assertVisitWithInvalidNowFails("music.field1 > now() - 300 and video.field1 > now() and music.field2 > now() - 300", "Specifying multiple document types is not allowed");
+        assertVisitWithInvalidNowFails("now() > music.field", "Left hand side of comparison must be a document field");
+    }
+
+    public void assertThatQueriesAreCreated(String selection, List<String> expectedDoctypes, List<String> expectedQueries) throws ParseException {
+        DocumentSelector selector = new DocumentSelector(selection);
+        NowCheckVisitor visitor = new NowCheckVisitor();
+        selector.visit(visitor);
+        assertTrue(visitor.requiresConversion());
+        SelectionExpressionConverter converter = new SelectionExpressionConverter();
+        selector.visit(converter);
+        Map<String, String> queryMap = converter.getQueryMap();
+        assertEquals(expectedQueries.size(), queryMap.size());
+        for (int i = 0; i < expectedQueries.size(); i++) {
+            assertTrue(queryMap.containsKey(expectedDoctypes.get(i)));
+            assertEquals(expectedQueries.get(i), queryMap.get(expectedDoctypes.get(i)));
+        }
+    }
+
+    public void assertVisitWithoutNowWorks(String expression) throws ParseException {
+        DocumentSelector selector = new DocumentSelector(expression);
+        NowCheckVisitor visitor = new NowCheckVisitor();
+        selector.visit(visitor);
+        assertFalse(visitor.requiresConversion());
+    }
+
+    public void assertVisitWithValidNowWorks(String expression) throws ParseException {
+        DocumentSelector selector = new DocumentSelector(expression);
+        NowCheckVisitor visitor = new NowCheckVisitor();
+        selector.visit(visitor);
+        assertTrue(visitor.requiresConversion());
+        SelectionExpressionConverter converter = new SelectionExpressionConverter();
+        try {
+            selector.visit(converter);
+        } catch (Exception e) {
+            assertFalse("Converter throws exception : " + e.getMessage(), true);
+        }
+    }
+
+    public void assertVisitWithInvalidNowFails(String expression, String expectedError) throws ParseException {
+        DocumentSelector selector = new DocumentSelector(expression);
+        NowCheckVisitor visitor = new NowCheckVisitor();
+        selector.visit(visitor);
+        assertTrue(visitor.requiresConversion());
+        SelectionExpressionConverter converter = new SelectionExpressionConverter();
+        try {
+            selector.visit(converter);
+            assertFalse("Should not be able to convert " + expression + " query", true);
+        } catch (Exception e) {
+            assertEquals(expectedError, e.getMessage());
+        }
+    }
+
+    private static DocumentPut createDocument(String id, Integer hInt, float hFloat, String hString, String content) {
+        Document doc = new Document(manager.getDocumentType("test"), new DocumentId(id));
+        if (hInt != null)
+            doc.setFieldValue("hint", new IntegerFieldValue(hInt));
+        doc.setFieldValue("hfloat", new FloatFieldValue(hFloat));
+        if (hString != null)
+            doc.setFieldValue("hstring", new StringFieldValue(hString));
+        doc.setFieldValue("content", new StringFieldValue(content));
+        return new DocumentPut(doc);
+    }
+
+    private static void assertParse(String expression) throws ParseException {
+        assertParse(expression, expression);
+    }
+
+    private static void assertParse(String expectedString, String expressionString) throws ParseException {
+        DocumentSelector selector = new DocumentSelector(expressionString);
+        if (expectedString != null) {
+            assertEquals(expectedString, selector.toString());
+        }
+    }
+
+    private static void assertParseError(String expressionString, String expectedError) {
+        try {
+            new DocumentSelector(expressionString);
+            fail("The expression '" + expressionString + "' should throw an exception.");
+        }
+        catch (ParseException e) {
+            Throwable t = e;
+            if (t.getCause() instanceof TokenMgrError) {
+                t = t.getCause();
+            }
+            assertEquals(expectedError, Exceptions.toMessageString(t).substring(0, expectedError.length()));
+        }
+    }
+
+    private static Result evaluate(String expressionString, DocumentOperation op) throws ParseException {
+        return new DocumentSelector(expressionString).accepts(op);
+    }
+
+    private static void assertError(String expressionString, DocumentOperation op, String expectedError) {
+        try {
+            evaluate(expressionString, op);
+            fail("The evaluation of '" + expressionString + "' should throw an exception.");
+        } catch (ParseException e) {
+            fail("The expression '" + expressionString + "' should assertEquals ok.");
+        } catch (RuntimeException e) {
+            System.err.println("Error was : " + e);
+            assertTrue(e.getMessage().length() >= expectedError.length());
+            assertEquals(expectedError, e.getMessage().substring(0, expectedError.length()));
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/select/OrderingSpecificationTestCase.java b/document/src/test/java/com/yahoo/document/select/OrderingSpecificationTestCase.java
new file mode 100644
index 00000000000..e47fe4e993e
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/select/OrderingSpecificationTestCase.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.select;
+
+import com.yahoo.document.select.*;
+
+/**
+ * Date: Sep 6, 2007
+ *
+ * @author <a href="mailto:humbe@yahoo-inc.com">H&aring;kon Humberset</a>
+ */
+public class OrderingSpecificationTestCase extends junit.framework.TestCase {
+
+    public OrderingSpecificationTestCase(String name) {
+        super(name);
+    }
+
+    public void testExpressions() throws Exception {
+        assertSelection("id.order(10,10) < 100", OrderingSpecification.DESCENDING,
+                        new OrderingSpecification(OrderingSpecification.DESCENDING, (long)99, (short)10, (short)10));
+        assertSelection("id.order(10,10) <= 100", OrderingSpecification.DESCENDING,
+                        new OrderingSpecification(OrderingSpecification.DESCENDING, (long)100, (short)10, (short)10));
+        assertSelection("id.order(10,10) > 100", OrderingSpecification.DESCENDING, null);
+        assertSelection("id.order(10,10) > 100", OrderingSpecification.ASCENDING,
+                        new OrderingSpecification(OrderingSpecification.ASCENDING, (long)101, (short)10, (short)10));
+        assertSelection("id.user==1234 AND id.order(10,10) > 100", OrderingSpecification.ASCENDING,
+                        new OrderingSpecification(OrderingSpecification.ASCENDING, (long)101, (short)10, (short)10));
+        assertSelection("id.order(10,10) >= 100", OrderingSpecification.ASCENDING,
+                        new OrderingSpecification(OrderingSpecification.ASCENDING, (long)100, (short)10, (short)10));
+        assertSelection("id.order(10,10) == 100", OrderingSpecification.ASCENDING,
+                        new OrderingSpecification(OrderingSpecification.ASCENDING, (long)100, (short)10, (short)10));
+        assertSelection("id.order(10,10) = 100", OrderingSpecification.DESCENDING,
+                        new OrderingSpecification(OrderingSpecification.DESCENDING, (long)100, (short)10, (short)10));
+        assertSelection("id.order(10,10) > 30 AND id.order(10,10) < 100", OrderingSpecification.ASCENDING,
+                        new OrderingSpecification(OrderingSpecification.ASCENDING, (long)31, (short)10, (short)10));
+        assertSelection("id.order(10,10) > 30 AND id.order(10,10) < 100", OrderingSpecification.DESCENDING,
+                        new OrderingSpecification(OrderingSpecification.DESCENDING, (long)99, (short)10, (short)10));
+        assertSelection("id.order(10,10) > 30 OR id.order(10,10) > 70", OrderingSpecification.ASCENDING,
+                        new OrderingSpecification(OrderingSpecification.ASCENDING, (long)31, (short)10, (short)10));
+        assertSelection("id.order(10,10) < 30 OR id.order(10,10) < 70", OrderingSpecification.DESCENDING,
+                        new OrderingSpecification(OrderingSpecification.DESCENDING, (long)69, (short)10, (short)10));
+    }
+
+    public void assertSelection(String selection, int ordering, OrderingSpecification wanted) throws Exception {
+        DocumentSelector selector = new DocumentSelector(selection);
+        if (wanted != null) {
+            assertEquals(wanted, selector.getOrdering(ordering));
+        } else {
+            assertNull(selector.getOrdering(ordering));
+        }
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/PredicateFieldValueSerializationTestCase.java b/document/src/test/java/com/yahoo/document/serialization/PredicateFieldValueSerializationTestCase.java
new file mode 100644
index 00000000000..eb4e06dc149
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/PredicateFieldValueSerializationTestCase.java
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.DataType;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.predicate.BooleanPredicate;
+import com.yahoo.document.predicate.Conjunction;
+import com.yahoo.document.predicate.Disjunction;
+import com.yahoo.document.predicate.FeatureRange;
+import com.yahoo.document.predicate.FeatureSet;
+import com.yahoo.document.predicate.Negation;
+import com.yahoo.document.predicate.Predicate;
+import org.junit.Test;
+import java.io.IOException;
+
+import static com.yahoo.document.serialization.SerializationTestUtils.deserializeDocument;
+import static com.yahoo.document.serialization.SerializationTestUtils.serializeDocument;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class PredicateFieldValueSerializationTestCase {
+
+    private final static String PREDICATE_FIELD = "my_predicate";
+    private final static String PREDICATE_FILES = "src/test/resources/predicates/";
+    private final static TestDocumentFactory docFactory =
+            new TestDocumentFactory(createDocType(), "id:test:my_type::foo");
+
+    private static DocumentType createDocType() {
+        DocumentType type = new DocumentType("my_type");
+        type.addField(PREDICATE_FIELD, DataType.PREDICATE);
+        return type;
+    }
+
+    @Test
+    public void requireThatPredicateFieldValuesAreDeserialized() {
+        Document prevDocument = docFactory.createDocument();
+        PredicateFieldValue prevPredicate = new PredicateFieldValue(new Conjunction(new FeatureSet("foo", "bar"),
+                                                                                    new FeatureRange("baz", 6L, 9L)));
+        prevDocument.setFieldValue(PREDICATE_FIELD, prevPredicate);
+        byte[] buf = serializeDocument(prevDocument);
+        Document nextDocument = deserializeDocument(buf, docFactory);
+        assertEquals(prevDocument, nextDocument);
+        assertEquals(prevPredicate, nextDocument.getFieldValue(PREDICATE_FIELD));
+    }
+
+    @Test
+    public void requireThatPredicateDeserializationMatchesCpp() throws IOException {
+        assertDeserialize("foo_in_bar_and_baz_in_cox", new Conjunction(new FeatureSet("foo", "bar"),
+                                                                       new FeatureSet("baz", "cox")));
+        assertDeserialize("foo_in_bar_or_baz_in_cox", new Disjunction(new FeatureSet("foo", "bar"),
+                                                                      new FeatureSet("baz", "cox")));
+        assertDeserialize("foo_in_6_9", new FeatureRange("foo", 6L, 9L));
+        assertDeserialize("foo_in_6_x", new FeatureRange("foo", 6L, null));
+        assertDeserialize("foo_in_x_9", new FeatureRange("foo", null, 9L));
+        assertDeserialize("foo_in_x_x", new FeatureRange("foo", null, null));
+        assertDeserialize("foo_in_x", new FeatureSet("foo"));
+        assertDeserialize("foo_in_bar", new FeatureSet("foo", "bar"));
+        assertDeserialize("foo_in_bar_baz", new FeatureSet("foo", "bar", "baz"));
+        assertDeserialize("not_foo_in_bar", new Negation(new FeatureSet("foo", "bar")));
+        assertDeserialize("true", new BooleanPredicate(true));
+        assertDeserialize("false", new BooleanPredicate(false));
+    }
+
+    private static void assertDeserialize(String fileName, Predicate expected) throws IOException {
+        Document document = docFactory.createDocument();
+        document.setFieldValue(PREDICATE_FIELD, new PredicateFieldValue(expected));
+        SerializationTestUtils.assertSerializationMatchesCpp(PREDICATE_FILES, fileName, document, docFactory);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializationHelperTestCase.java b/document/src/test/java/com/yahoo/document/serialization/SerializationHelperTestCase.java
new file mode 100644
index 00000000000..fbd3ea3afdc
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/SerializationHelperTestCase.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.datatypes.Raw;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.text.Utf8;
+import com.yahoo.text.Utf8Array;
+import com.yahoo.vespa.objects.BufferSerializer;
+
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SerializationHelperTestCase extends junit.framework.TestCase {
+    public SerializationHelperTestCase(String name) {
+        super(name);
+    }
+
+    public void testGetNullTerminatedString() throws Exception {
+        //This is a test.0ab
+        byte[] test = {0x54, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x61, 0x20, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x0,
+                0x61, 0x62};
+
+        BufferSerializer data = BufferSerializer.wrap(test);
+
+        assertTrue(data.position() == 0);
+
+        Utf8Array thisIsATest = VespaDocumentDeserializer42.parseNullTerminatedString(data.getBuf().getByteBuffer());
+
+        assertTrue(thisIsATest.equals(new Utf8Array(Utf8.toBytes("This is a test."))));
+        assertTrue(data.position() == 16);
+        assertTrue(test[16] == 0x61); //a
+
+        data.position(0);
+
+        assertTrue(data.position() == 0);
+
+        Utf8Array thisIsATestAgain = VespaDocumentDeserializer42.parseNullTerminatedString(data.getBuf().getByteBuffer(), 15);
+
+        assertTrue(thisIsATestAgain.equals(new Utf8Array(Utf8.toBytes("This is a test."))));
+        assertTrue(data.position() == 16);
+        assertTrue(test[16] == 0x61); //a
+    }
+
+    public void testSerializeRawField() throws UnsupportedEncodingException {
+        GrowableByteBuffer gbuf = new GrowableByteBuffer();
+        ByteBuffer rawValue = ByteBuffer.wrap(Utf8.toBytes("0123456789"));
+        rawValue.position(7);
+        Raw value = new Raw(rawValue);
+        value.serialize(gbuf);
+
+        assertEquals(7, gbuf.position());
+        assertEquals(7, rawValue.position());
+
+        value = new Raw(rawValue);
+        value.serialize(gbuf);
+
+        assertEquals(14, gbuf.position());
+        assertEquals(7, rawValue.position());
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java
new file mode 100644
index 00000000000..18e517fb74e
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.Document;
+import com.yahoo.io.GrowableByteBuffer;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Helper class with utils used in serialization and deserialization test cases.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class SerializationTestUtils {
+
+    public static byte[] serializeDocument(Document doc) {
+        GrowableByteBuffer out = new GrowableByteBuffer();
+        DocumentSerializerFactory.create42(out).write(doc);
+        out.flip();
+        byte[] buf = new byte[out.remaining()];
+        out.get(buf);
+        return buf;
+    }
+
+    public static Document deserializeDocument(byte[] buf, TestDocumentFactory factory) {
+        Document document = factory.createDocument();
+        DocumentDeserializerFactory.create42(factory.typeManager(), new GrowableByteBuffer(ByteBuffer.wrap(buf))).read(document);
+        return document;
+    }
+
+    public static void assertSerializationMatchesCpp(String binaryFilesFolder, String fileName,
+                                                     Document document, TestDocumentFactory factory) throws IOException {
+        byte[] buf = serializeDocument(document);
+        Files.write(Paths.get(binaryFilesFolder, fileName + "__java"), buf,
+                StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
+
+        assertDeserializeFromFile(Paths.get(binaryFilesFolder, fileName + "__java"), document, factory);
+        assertDeserializeFromFile(Paths.get(binaryFilesFolder, fileName + "__cpp"), document, factory);
+    }
+
+    private static void assertDeserializeFromFile(Path path, Document document, TestDocumentFactory factory) throws IOException {
+        byte[] buf = Files.readAllBytes(path);
+        Document deserializedDocument = deserializeDocument(buf, factory);
+        assertEquals(path.toString(), document, deserializedDocument);
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java b/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java
new file mode 100644
index 00000000000..d8ed160e1c4
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java
@@ -0,0 +1,213 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentTypeManagerConfigurer;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.annotation.*;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.io.GrowableByteBuffer;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.ReadableByteChannel;
+import java.util.ArrayList;
+
+/**
+ * Tests that serialization of annotations from Java generates the
+ * output expected by the C++ deserialization system.
+ *
+ * If the format changes and this test starts failing, you should
+ * generate new golden files to check against. This will cause the C++
+ * test to fail, so you will need to update the
+ * AnnotationDeserialization component to handle the format changes.
+ */
+public class SerializeAnnotationsTestCase extends junit.framework.TestCase {
+    private static final String PATH = "src/tests/serialization/";
+    DocumentTypeManager docMan = new DocumentTypeManager();
+
+    @Override
+    public void setUp() {
+        DocumentTypeManagerConfigurer.configure(docMan,
+                                                "file:src/tests/serialization/" +
+                                                "annotation.serialize.test.cfg");
+    }
+
+    public void testSerializeSimpleTree() throws IOException {
+        SpanList root = new SpanList();
+        root.add(new Span(0, 19))
+            .add(new Span(19, 5))
+            .add(new Span(24, 21))
+            .add(new Span(45, 23))
+            .add(new Span(68, 14));
+        SpanTree tree = new SpanTree("html", root);
+        StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj " +
+                                                      "lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj l jlkj lkj lkj " +
+                                                      "lkjoijoij oij oij oij oij oij oijoijoij oij oij oij oij oij " +
+                                                      "oijoijoijoijoijoijoijoijoijoijoijoijoij oij oij oij oij " +
+                                                      "oijaosdifjoai fdoais jdoasi jai os oafoai ai dfojsfoa dfoi dsf" +
+                                                      "aosifjasofija sodfij oasdifj aosdiosifjsi ooai oais osi");
+        value.setSpanTree(tree);
+
+        /*
+        Important note! The iteration order of annotations in SpanTree.iterator() is non-deterministic, meaning
+        that the order which annotations are serialized will differ between test runs. Thus, we cannot assert
+        that a serialized buffer is equal to a buffer written earlier. We can, however, assert that the size stays
+        the same, and the deserialized values from the buffers should be equal.
+         */
+
+        //important! call readFile() before writeFile()!
+        ByteBuffer serializedFromFile = readFile("test_data_serialized_simple");
+        ByteBuffer serialized = writeFile(value, "test_data_serialized_simple");
+        assertEquals(serialized.limit(), serializedFromFile.limit());
+
+        StringFieldValue valueFromFile = new StringFieldValue();
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(docMan, new GrowableByteBuffer(serializedFromFile));
+        deserializer.read(null, valueFromFile);
+        assertEquals(value, valueFromFile);
+    }
+
+    public void testSerializeAdvancedTree() throws IOException {
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree("html", root);
+
+        DataType positionType = docMan.getDataType("myposition");
+        StructDataType cityDataType =
+            (StructDataType) docMan.getDataType("annotation.city");
+
+        AnnotationTypeRegistry registry = docMan.getAnnotationTypeRegistry();
+        AnnotationType textType = registry.getType("text");
+        AnnotationType beginTag = registry.getType("begintag");
+        AnnotationType endTag = registry.getType("endtag");
+        AnnotationType bodyType = registry.getType("body");
+        AnnotationType paragraphType = registry.getType("paragraph");
+        AnnotationType cityType = registry.getType("city");
+
+        AnnotationReferenceDataType annRefType =
+            (AnnotationReferenceDataType)
+            docMan.getDataType("annotationreference<text>");
+
+        Struct position = new Struct(positionType);
+        position.setFieldValue("latitude", new DoubleFieldValue(37.774929));
+        position.setFieldValue("longitude", new DoubleFieldValue(-122.419415));
+
+        Annotation sanAnnotation = new Annotation(textType);
+        Annotation franciscoAnnotation = new Annotation(textType);
+
+        Struct positionWithRef = cityDataType.createFieldValue();
+        positionWithRef.setFieldValue("position", position);
+
+        Field referencesField = cityDataType.getField("references");
+        Array<FieldValue> refList =
+            new Array<FieldValue>(referencesField.getDataType());
+        refList.add(new AnnotationReference(annRefType, sanAnnotation));
+        refList.add(new AnnotationReference(annRefType, franciscoAnnotation));
+        positionWithRef.setFieldValue(referencesField, refList);
+
+        Annotation city = new Annotation(cityType, positionWithRef);
+
+        AlternateSpanList paragraph = new AlternateSpanList();
+        paragraph.addChildren(new ArrayList<SpanNode>(), 0);
+        paragraph.setProbability(0, 0.9);
+        paragraph.setProbability(1, 0.1);
+        {
+            Span span1 = new Span(6, 3);
+            Span span2 = new Span(9, 10);
+            Span span3 = new Span(19, 4);
+            Span span4 = new Span(23, 4);
+            paragraph.add(0, span1)
+                .add(0, span2)
+                .add(0, span3)
+                .add(0, span4);
+
+            Span alt_span1 = new Span(6, 13);
+            Span alt_span2 = new Span(19, 8);
+            paragraph.add(1, alt_span1)
+                .add(1, alt_span2);
+
+            tree.annotate(span1, beginTag)
+                .annotate(span2, textType)
+                .annotate(span3, sanAnnotation)
+                .annotate(span4, endTag)
+                .annotate(alt_span1, textType)
+                .annotate(alt_span2, bodyType)
+                .annotate(paragraph, paragraphType);
+        }
+
+        {
+            Span span1 = new Span(0, 6);
+            Span span2 = new Span(27, 9);
+            Span span3 = new Span(36, 8);
+            root.add(span1)
+                .add(paragraph)
+                .add(span2)
+                .add(span3);
+
+            tree.annotate(span1, beginTag)
+                .annotate(span2, franciscoAnnotation)
+                .annotate(span3, endTag)
+                .annotate(root, bodyType)
+                .annotate(city);
+        }
+
+        StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj " +
+                                                      "lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj l jlkj lkj lkj " +
+                                                      "lkjoijoij oij oij oij oij oij oijoijoij oij oij oij oij oij " +
+                                                      "oijoijoijoijoijoijoijoijoijoijoijoijoij oij oij oij oij " +
+                                                      "oijaosdifjoai fdoais jdoasi jai os oafoai ai dfojsfoa dfoi dsf" +
+                                                      "aosifjasofija sodfij oasdifj aosdiosifjsi ooai oais osi");
+        value.setSpanTree(tree);
+
+        //important! call readFile() before writeFile()!
+        ByteBuffer serializedFromFile = readFile("test_data_serialized_advanced");
+        ByteBuffer serialized = writeFile(value, "test_data_serialized_advanced");
+        assertEquals(serialized.limit(), serializedFromFile.limit());
+
+        StringFieldValue valueFromFile = new StringFieldValue();
+        DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(docMan, new GrowableByteBuffer(serializedFromFile));
+        deserializer.read(null, valueFromFile);
+        assertEquals(value, valueFromFile);
+    }
+
+    private static ByteBuffer writeFile(StringFieldValue value, String fileName) throws IOException {
+        fileName = PATH + fileName;
+
+        //serialize our tree to buffer
+        VespaDocumentSerializer42 serializer = new VespaDocumentSerializer42();
+        serializer.write(null, value);
+        ByteBuffer serializedBuf = serializer.getBuf().getByteBuffer();
+        serializedBuf.flip();
+
+        //write our tree to disk
+        File file = new File(fileName);
+        FileChannel wChannel = new FileOutputStream(file, false).getChannel();
+        wChannel.write(serializedBuf);
+        wChannel.close();
+
+        serializedBuf.position(0);
+        return serializedBuf;
+    }
+
+    private static ByteBuffer readFile(String fileName) throws IOException {
+        fileName = PATH + fileName;
+
+        //read tree from disk
+        ReadableByteChannel channel = new FileInputStream(fileName).getChannel();
+        ByteBuffer readBuf = ByteBuffer.allocate(4096);
+        channel.read(readBuf);
+        readBuf.flip();
+        channel.close();
+
+        return readBuf;
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java b/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java
new file mode 100644
index 00000000000..62f9ea9f4d3
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/TensorFieldValueSerializationTestCase.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.TensorFieldValue;
+import com.yahoo.tensor.MapTensor;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static com.yahoo.document.serialization.SerializationTestUtils.deserializeDocument;
+import static com.yahoo.document.serialization.SerializationTestUtils.serializeDocument;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class TensorFieldValueSerializationTestCase {
+
+    private final static String TENSOR_FIELD = "my_tensor";
+    private final static String TENSOR_FILES = "src/test/resources/tensor/";
+    private final static TestDocumentFactory docFactory =
+            new TestDocumentFactory(createDocType(), "id:test:my_type::foo");
+
+    private static DocumentType createDocType() {
+        DocumentType type = new DocumentType("my_type");
+        type.addField(TENSOR_FIELD, DataType.TENSOR);
+        return type;
+    }
+
+    @Test
+    public void requireThatTensorFieldValueIsSerializedAndDeserialized() {
+        assertSerialization(new TensorFieldValue());
+        assertSerialization(createTensor("{}"));
+        assertSerialization(createTensor("{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e}:5.0}"));
+    }
+
+    @Test
+    public void requireThatSerializationMatchesCpp() throws IOException {
+        assertSerializationMatchesCpp("non_existing_tensor", new TensorFieldValue());
+        assertSerializationMatchesCpp("empty_tensor", createTensor("{}"));
+        assertSerializationMatchesCpp("multi_cell_tensor",
+                createTensor("{{dimX:a,dimY:bb}:2.0,{dimX:ccc,dimY:dddd}:3.0,{dimX:e}:5.0}"));
+    }
+
+    private static void assertSerialization(TensorFieldValue tensor) {
+        Document document = docFactory.createDocument();
+        document.setFieldValue(TENSOR_FIELD, tensor);
+        byte[] buf = serializeDocument(document);
+        Document deserializedDocument = deserializeDocument(buf, docFactory);
+        assertEquals(document, deserializedDocument);
+        assertEquals(tensor, deserializedDocument.getFieldValue(TENSOR_FIELD));
+    }
+
+    private static void assertSerializationMatchesCpp(String fileName, TensorFieldValue tensor) throws IOException {
+        Document document = docFactory.createDocument();
+        document.setFieldValue(TENSOR_FIELD, tensor);
+        SerializationTestUtils.assertSerializationMatchesCpp(TENSOR_FILES, fileName, document, docFactory);
+    }
+
+    private static TensorFieldValue createTensor(String tensor) {
+        return new TensorFieldValue(MapTensor.from(tensor));
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/TestDocumentFactory.java b/document/src/test/java/com/yahoo/document/serialization/TestDocumentFactory.java
new file mode 100644
index 00000000000..d54562862b7
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/TestDocumentFactory.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+
+/**
+ * Helper class for creating a document for a given document type.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class TestDocumentFactory {
+
+    private final DocumentTypeManager typeManager = new DocumentTypeManager();
+    private final DocumentType docType;
+    private final String defaultId;
+
+    public TestDocumentFactory(DocumentType docType, String defaultId) {
+        this.docType = docType;
+        this.defaultId = defaultId;
+        typeManager.register(docType);
+    }
+
+    public Document createDocument(String id) {
+        return new Document(docType, id);
+    }
+
+    public Document createDocument() {
+        return createDocument(defaultId);
+    }
+
+    public DocumentTypeManager typeManager() {
+        return typeManager;
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java
new file mode 100644
index 00000000000..823bb0b196d
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.io.GrowableByteBuffer;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class VespaDocumentSerializerTestCase {
+
+    @Test
+    public void requireThatGetSerializedSizeUsesLatestSerializer() {
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField("my_str", DataType.STRING);
+        docType.addField("my_int", DataType.INT);
+        Document doc = new Document(docType, "doc:scheme:");
+        doc.setFieldValue("my_str", new StringFieldValue("foo"));
+        doc.setFieldValue("my_int", new IntegerFieldValue(69));
+
+        GrowableByteBuffer buf = new GrowableByteBuffer();
+        doc.serialize(buf);
+        assertEquals(buf.position(), VespaDocumentSerializer42.getSerializedSize(doc));
+    }
+
+    @Test
+    public void requireThatPredicateFieldValuesAreSerialized() {
+        DocumentType docType = new DocumentType("my_type");
+        Field field = new Field("my_predicate", DataType.PREDICATE);
+        docType.addField(field);
+        Document doc = new Document(docType, "doc:scheme:");
+        PredicateFieldValue predicate = Mockito.mock(PredicateFieldValue.class);
+        doc.setFieldValue("my_predicate", predicate);
+
+        DocumentSerializerFactory.create42(new GrowableByteBuffer()).write(doc);
+        Mockito.verify(predicate, Mockito.times(1)).serialize(Mockito.same(field), Mockito.any(FieldWriter.class));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/XmlDocumentWriterTestCase.java b/document/src/test/java/com/yahoo/document/serialization/XmlDocumentWriterTestCase.java
new file mode 100644
index 00000000000..b185250cbd8
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/XmlDocumentWriterTestCase.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class XmlDocumentWriterTestCase {
+
+    @Test
+    public void requireThatPredicateFieldValuesAreSerializedAsString() {
+        DocumentType docType = new DocumentType("my_type");
+        Field field = new Field("my_predicate", DataType.PREDICATE);
+        docType.addField(field);
+        Document doc = new Document(docType, "doc:scheme:");
+        PredicateFieldValue predicate = Mockito.mock(PredicateFieldValue.class);
+        doc.setFieldValue("my_predicate", predicate);
+
+        new XmlDocumentWriter().write(doc);
+        Mockito.verify(predicate, Mockito.times(1)).printXml(Mockito.any(XmlStream.class));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/document/serialization/XmlStreamTestCase.java b/document/src/test/java/com/yahoo/document/serialization/XmlStreamTestCase.java
new file mode 100644
index 00000000000..57a7fc76427
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/serialization/XmlStreamTestCase.java
@@ -0,0 +1,104 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.serialization;
+
+import com.yahoo.document.serialization.XmlStream;
+
+import java.io.IOException;
+
+/**
+ * Test for XmlStream used in document XML serialization.
+ *
+ * @author <a href="humbe@yahoo-inc.com>Haakon Humberset</a>
+ */
+public class XmlStreamTestCase extends junit.framework.TestCase {
+
+    public XmlStreamTestCase(String name) {
+        super(name);
+    }
+
+    /** A catch all test checking that regular usage looks good. */
+    public void testNormalUsage() {
+        XmlStream xml = new XmlStream();
+        xml.setIndent("  ");
+        xml.beginTag("foo");
+        xml.addAttribute("bar", "foo");
+        xml.addAttribute("foo", "bar");
+        xml.addContent("foo");
+        xml.beginTag("bar");
+        xml.endTag();
+        xml.beginTag("foo");
+        xml.beginTag("bar");
+        xml.addAttribute("foo", "bar");
+        xml.addContent("bar");
+        xml.endTag();
+        xml.endTag();
+        xml.addContent("bar");
+        xml.beginTag("foo");
+        xml.addContent("foo");
+        xml.addContent("bar");
+        xml.endTag();
+        xml.endTag();
+        String expected =
+              "<foo bar=\"foo\" foo=\"bar\">\n"
+            + "  foo\n"
+            + "  <bar/>\n"
+            + "  <foo>\n"
+            + "    <bar foo=\"bar\">bar</bar>\n"
+            + "  </foo>\n"
+            + "  bar\n"
+            + "  <foo>foobar</foo>\n"
+            + "</foo>\n";
+        assertEquals(expected, xml.toString());
+    }
+
+    /**
+     * Test that XML tag and attribute names are checked for validity.
+     * Only the obvious illegal characters are tested currently.
+     */
+    public void testIllegalAttributeNames() {
+        String illegalNames[] = {">foo", "foo<bar", " foo", "bar ", "foo bar", "foo\"bar", "&foo"};
+        XmlStream xml = new XmlStream();
+        for (String name : illegalNames) {
+            try {
+                xml.beginTag(name);
+                assertTrue(false);
+            } catch (IllegalArgumentException e) {
+                assertTrue(e.getMessage().indexOf("cannot be used as an XML tag name") != -1);
+            }
+        }
+        xml.beginTag("test");
+        for (String name : illegalNames) {
+            try {
+                xml.addAttribute(name, "");
+                assertTrue(false);
+            } catch (IllegalArgumentException e) {
+                assertTrue(e.getMessage().indexOf("cannot be used as an XML attribute name") != -1);
+            }
+        }
+    }
+
+    /** Test that XML attribute values are XML escaped. */
+    public void testAttributeEscaping() {
+        //String badString = "\"&<>'\n\r\u0000";
+        String badString = "\"&<>'";
+        XmlStream xml = new XmlStream();
+        xml.beginTag("foo");
+        xml.addAttribute("bar", badString);
+        xml.endTag();
+        String expected = "<foo bar=\"&quot;&amp;&lt;&gt;'\"/>\n";
+        assertEquals(expected, xml.toString());
+    }
+
+    /** Test that content is XML escaped. */
+    public void testContentEscaping() {
+        //String badString = "\"&<>'\n\r\u0000";
+        String badString = "\"&<>'";
+        XmlStream xml = new XmlStream();
+        xml.beginTag("foo");
+        xml.addContent(badString);
+        xml.endTag();
+        String expected = "<foo>\"&amp;&lt;&gt;'</foo>\n";
+        assertEquals(expected, xml.toString());
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/update/FieldUpdateTestCase.java b/document/src/test/java/com/yahoo/document/update/FieldUpdateTestCase.java
new file mode 100644
index 00000000000..e14922eb2df
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/update/FieldUpdateTestCase.java
@@ -0,0 +1,368 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.update;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.serialization.DocumentDeserializerFactory;
+import com.yahoo.document.serialization.DocumentSerializer;
+import com.yahoo.document.serialization.DocumentSerializerFactory;
+
+import java.util.List;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class FieldUpdateTestCase extends junit.framework.TestCase {
+    Field strfoo;
+    Field strarray;
+    Field strws;
+    Field strws2;
+    DocumentTypeManager docman = new DocumentTypeManager();
+
+    public void setUp() {
+        docman = new DocumentTypeManager();
+
+        DocumentType testType = new DocumentType("foobar");
+        testType.addField(new Field("strfoo", DataType.STRING));
+        testType.addField(new Field("strarray", DataType.getArray(DataType.STRING)));
+        testType.addField(new Field("strws", DataType.getWeightedSet(DataType.STRING)));
+        testType.addField(new Field("strws2", DataType.getWeightedSet(DataType.STRING, true, true)));
+        docman.registerDocumentType(testType);
+
+
+        strfoo = docman.getDocumentType("foobar").getField("strfoo");
+        strarray = docman.getDocumentType("foobar").getField("strarray");
+        strws = docman.getDocumentType("foobar").getField("strws");
+        strws2 = docman.getDocumentType("foobar").getField("strws2");
+    }
+
+    public void testInstantiationExceptions() {
+        //add(field, value)
+        try {
+            FieldUpdate.createAdd(strfoo, new StringFieldValue("banana"));
+            fail("Should have gotten exception");
+        } catch (UnsupportedOperationException uoe) {}
+        FieldUpdate.createAdd(strarray, new StringFieldValue("banana"));
+        try {
+            FieldUpdate.createAdd(strarray, new Array(DataType.getArray(DataType.STRING)));
+                    fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+        FieldUpdate.createAdd(strws, new StringFieldValue("banana"));
+        try {
+            FieldUpdate.createAdd(strws, new Array(DataType.getArray(DataType.STRING)));
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+
+
+        //add(field, key, weight)
+        try {
+            FieldUpdate.createAdd(strfoo, new StringFieldValue("apple"), 5);
+            fail("Should have gotten exception");
+        } catch (UnsupportedOperationException uoe) {}
+        FieldUpdate.createAdd(strarray, new StringFieldValue("apple"), 5);
+        FieldUpdate.createAdd(strws, new StringFieldValue("apple"), 5);
+        try {
+            FieldUpdate.createAdd(strws, new Array(DataType.getArray(DataType.STRING)), 50);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+
+
+        Array<StringFieldValue> fruitList = new Array<>(DataType.getArray(DataType.STRING));
+        fruitList.add(new StringFieldValue("kiwi"));
+        fruitList.add(new StringFieldValue("mango"));
+        Array<Raw> rawList = new Array<>(DataType.getArray(DataType.RAW));
+        rawList.add(new Raw());
+        rawList.add(new Raw());
+        rawList.add(new Raw());
+
+
+        //addall(field, list)
+        try {
+            FieldUpdate.createAddAll(strfoo, fruitList);
+            fail("Should have gotten exception");
+        } catch (UnsupportedOperationException uoe) {}
+        FieldUpdate.createAddAll(strarray, fruitList);
+        try {
+            FieldUpdate.createAddAll(strarray, rawList);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+        FieldUpdate.createAddAll(strws, fruitList);
+
+        WeightedSet fruitWs = new WeightedSet(DataType.getWeightedSet(DataType.STRING));
+        fruitWs.put(new StringFieldValue("pineapple"), 50);
+        fruitWs.put(new StringFieldValue("grape"), 10);
+        WeightedSet rawWs = new WeightedSet(DataType.getWeightedSet(DataType.RAW));
+        rawWs.put(new Raw(), 5);
+        rawWs.put(new Raw(), 5);
+        rawWs.put(new Raw(), 4);
+
+
+        //addall(field, weightedset)
+        try {
+            FieldUpdate.createAddAll(strfoo, fruitWs);
+            fail("Should have gotten exception");
+        } catch (UnsupportedOperationException uoe) {}
+        FieldUpdate.createAddAll(strarray, fruitWs);
+        FieldUpdate.createAddAll(strws, fruitWs);
+        try {
+            FieldUpdate.createAddAll(strws, rawWs);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+
+
+        //assign(field, object)
+        FieldUpdate.createAssign(strfoo, new StringFieldValue("potato"));
+        FieldUpdate.createAssign(strarray, fruitList);
+        FieldUpdate.createAssign(strws, fruitWs);
+        try {
+            FieldUpdate.createAssign(strfoo, new IntegerFieldValue(69));
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException uoe) {}
+
+
+        //decrement(field, object, decrement)
+        try {
+            FieldUpdate.createDecrement(strfoo, new StringFieldValue("ruccola"), 49d);
+            fail("Should have gotten exception");
+        } catch (UnsupportedOperationException uoe) {}
+        try {
+            FieldUpdate.createDecrement(strarray, new StringFieldValue("ruccola"), 49d);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+        FieldUpdate.createDecrement(strws, new StringFieldValue("ruccola"), 49d);
+        try {
+            FieldUpdate.createDecrement(strws, new Raw(), 48d);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+
+
+        //increment(field, object, increment)
+        try {
+            FieldUpdate.createIncrement(strfoo, new StringFieldValue("ruccola"), 49d);
+            fail("Should have gotten exception");
+        } catch (UnsupportedOperationException uoe) {}
+        try {
+            FieldUpdate.createIncrement(strarray, new StringFieldValue("ruccola"), 49d);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+        FieldUpdate.createIncrement(strws, new StringFieldValue("ruccola"), 49d);
+        try {
+            FieldUpdate.createIncrement(strws, new Raw(), 48d);
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+
+
+        //remove(field)
+        FieldUpdate.createClear(strfoo);
+        FieldUpdate.createClear(strarray);
+        FieldUpdate.createClear(strws);
+
+
+        //remove(field, object)
+        try {
+            FieldUpdate.createRemove(strfoo, new StringFieldValue("salad"));
+            fail("Should have gotten exception");
+        } catch (UnsupportedOperationException uoe) {}
+        FieldUpdate.createRemove(strarray, new StringFieldValue("salad"));
+        try {
+            FieldUpdate.createRemove(strarray, new Raw());
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+        FieldUpdate.createRemove(strws, new StringFieldValue("salad"));
+        try {
+            FieldUpdate.createRemove(strws, new Raw());
+            fail("Should have gotten exception");
+        } catch (IllegalArgumentException iae) {}
+    }
+
+    // Copy all field updates using serialization to verify that it is supported
+    private FieldUpdate serializedCopy(FieldUpdate source, DocumentType docType) {
+        DocumentSerializer buffer = DocumentSerializerFactory.create42();
+        source.serialize(buffer);
+        buffer.getBuf().flip();
+        FieldUpdate copy = new FieldUpdate(DocumentDeserializerFactory.create42(docman, buffer.getBuf()), docType, Document.SERIALIZED_VERSION);
+        assertEquals(source, copy);
+        return copy;
+    }
+
+    public void testApplyToSingleValue() {
+        Document testDoc = new Document(docman.getDocumentType("foobar"), new DocumentId("doc:test:ballooo"));
+        FieldUpdate alter = FieldUpdate.create(strfoo);
+
+        ValueUpdate assign = ValueUpdate.createAssign(new StringFieldValue("potato"));
+        alter.addValueUpdate(assign);
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(new StringFieldValue("potato"), testDoc.getFieldValue("strfoo"));
+
+        FieldUpdate clear = FieldUpdate.createClearField(strfoo);
+        clear = serializedCopy(clear, testDoc.getDataType());
+        clear.applyTo(testDoc);
+        assertNull(testDoc.getFieldValue("strfoo"));
+    }
+
+    public void testApplyToArray() {
+        Array<StringFieldValue> fruitList = new Array<>(DataType.getArray(DataType.STRING));
+        fruitList.add(new StringFieldValue("kiwi"));
+        fruitList.add(new StringFieldValue("mango"));
+        Document testDoc = new Document(docman.getDocumentType("foobar"), new DocumentId("doc:test:ballooo"));
+        FieldUpdate alter = FieldUpdate.create(strarray);
+
+        alter.addValueUpdate(ValueUpdate.createAdd(new StringFieldValue("banana")));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(1, ((List) testDoc.getFieldValue("strarray")).size());
+        assertEquals(new StringFieldValue("banana"), ((List) testDoc.getFieldValue("strarray")).get(0));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdates(ValueUpdate.createAddAll(fruitList));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(3, ((List) testDoc.getFieldValue("strarray")).size());
+        assertEquals(new StringFieldValue("banana"), ((List) testDoc.getFieldValue("strarray")).get(0));
+        assertEquals(new StringFieldValue("kiwi"), ((List) testDoc.getFieldValue("strarray")).get(1));
+        assertEquals(new StringFieldValue("mango"), ((List) testDoc.getFieldValue("strarray")).get(2));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createAssign(fruitList));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        System.err.println(testDoc.getFieldValue("strarray"));
+        assertEquals(2, ((List) testDoc.getFieldValue("strarray")).size());
+        assertEquals(new StringFieldValue("kiwi"), ((List) testDoc.getFieldValue("strarray")).get(0));
+        assertEquals(new StringFieldValue("mango"), ((List) testDoc.getFieldValue("strarray")).get(1));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createRemove(new StringFieldValue("kiwi")));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(1, ((List) testDoc.getFieldValue("strarray")).size());
+        assertEquals(new StringFieldValue("mango"), ((List) testDoc.getFieldValue("strarray")).get(0));
+
+        FieldUpdate clear = FieldUpdate.createClearField(strarray);
+        clear = serializedCopy(clear, testDoc.getDataType());
+        clear.applyTo(testDoc);
+        assertNull(testDoc.getFieldValue("strarray"));
+    }
+
+    public void testApplyToWs() {
+        WeightedSet fruitWs = new WeightedSet(DataType.getWeightedSet(DataType.STRING));
+        fruitWs.put(new StringFieldValue("pineapple"), 50);
+        fruitWs.put(new StringFieldValue("apple"), 10);
+        Document testDoc = new Document(docman.getDocumentType("foobar"), new DocumentId("doc:test:ballooo"));
+        FieldUpdate alter = FieldUpdate.create(strws);
+        FieldUpdate alter2 = FieldUpdate.create(strws2);
+
+
+        alter.addValueUpdate(ValueUpdate.createAdd(new StringFieldValue("banana")));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(1, ((WeightedSet) testDoc.getFieldValue("strws")).size());
+        assertEquals(1, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("banana")));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createAdd(new StringFieldValue("apple"), 5));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(2, ((WeightedSet) testDoc.getFieldValue("strws")).size());
+        assertEquals(1, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("banana")));
+        assertEquals(5, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("apple")));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdates(ValueUpdate.createAddAll(fruitWs));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(3, ((WeightedSet) testDoc.getFieldValue("strws")).size());
+        assertEquals(1, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("banana")));
+        assertEquals(10, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("apple")));
+        assertEquals(50, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("pineapple")));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createAssign(fruitWs));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(2, ((WeightedSet) testDoc.getFieldValue("strws")).size());
+        assertEquals(50, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("pineapple")));
+        assertEquals(10, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("apple")));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createDecrement(new StringFieldValue("pineapple"), 3d));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(2, ((WeightedSet) testDoc.getFieldValue("strws")).size());
+        assertEquals(47, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("pineapple")));
+        assertEquals(10, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("apple")));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createIncrement(new StringFieldValue("pineapple"), 13d));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(2, ((WeightedSet) testDoc.getFieldValue("strws")).size());
+        assertEquals(60, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("pineapple")));
+        assertEquals(10, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("apple")));
+
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createRemove(new StringFieldValue("apple")));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(1, ((WeightedSet) testDoc.getFieldValue("strws")).size());
+        assertEquals(60, (int) ((WeightedSet) testDoc.getFieldValue("strws")).get(new StringFieldValue("pineapple")));
+
+        // Test createifnonexistant
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createIncrement(new StringFieldValue("apple"), 1));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertNull(((WeightedSet)testDoc.getFieldValue("strws")).get(new StringFieldValue("apple")));
+
+        alter2.addValueUpdate(ValueUpdate.createIncrement(new StringFieldValue("apple"), 1));
+        alter2 = serializedCopy(alter2, testDoc.getDataType());
+        alter2.applyTo(testDoc);
+        assertEquals(1, ((WeightedSet) testDoc.getFieldValue("strws2")).size());
+        assertEquals(1, (int) ((WeightedSet) testDoc.getFieldValue("strws2")).get(new StringFieldValue("apple")));
+
+        // Test removeifzero
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createAdd(new StringFieldValue("banana")));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(1, (int) ((WeightedSet)testDoc.getFieldValue("strws")).get(new StringFieldValue("banana")));
+        alter.clearValueUpdates();
+        alter.addValueUpdate(ValueUpdate.createDecrement(new StringFieldValue("banana"), 1));
+        alter = serializedCopy(alter, testDoc.getDataType());
+        alter.applyTo(testDoc);
+        assertEquals(0, (int) ((WeightedSet)testDoc.getFieldValue("strws")).get(new StringFieldValue("banana")));
+
+        alter2.clearValueUpdates();
+        alter2.addValueUpdate(ValueUpdate.createAdd(new StringFieldValue("banana")));
+        alter2 = serializedCopy(alter2, testDoc.getDataType());
+        alter2.applyTo(testDoc);
+        assertEquals(1, (int) ((WeightedSet)testDoc.getFieldValue("strws2")).get(new StringFieldValue("banana")));
+
+        alter2.clearValueUpdates();
+        alter2.addValueUpdate(ValueUpdate.createDecrement(new StringFieldValue("banana"), 1));
+        alter2 = serializedCopy(alter2, testDoc.getDataType());
+        alter2.applyTo(testDoc);
+        assertNull(((WeightedSet)testDoc.getFieldValue("strws2")).get(new StringFieldValue("banana")));
+
+        FieldUpdate clear = FieldUpdate.createClearField(strws);
+        clear = serializedCopy(clear, testDoc.getDataType());
+        clear.applyTo(testDoc);
+        assertNull(testDoc.getFieldValue("strws"));
+    }
+
+    public void testArithmeticUpdatesOnAutoCreatedWSetItemsAreZeroBased() {
+        Document testDoc = new Document(
+                docman.getDocumentType("foobar"),
+                new DocumentId("doc:test:ballooo"));
+        // strws2 is fixture weightedset type with create-if-non-existing
+        // and remove-if-zero attributes set.
+        FieldUpdate update = FieldUpdate.create(strws2);
+        StringFieldValue key = new StringFieldValue("apple");
+        update.addValueUpdate(ValueUpdate.createIncrement(key, 1));
+        update.applyTo(testDoc);
+        assertEquals(1, ((WeightedSet)testDoc.getFieldValue("strws2")).size());
+        assertEquals(1, (int)((WeightedSet) testDoc.getFieldValue("strws2")).get(key));
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/update/SerializationTestCase.java b/document/src/test/java/com/yahoo/document/update/SerializationTestCase.java
new file mode 100644
index 00000000000..0d73138e077
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/update/SerializationTestCase.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.update;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.serialization.*;
+
+import java.io.FileOutputStream;
+
+/**
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class SerializationTestCase extends junit.framework.TestCase {
+
+    private DocumentType documentType;
+
+    private Field field;
+
+    public SerializationTestCase(String name) {
+        super(name);
+    }
+
+    public void setUp() {
+        documentType=new DocumentType("document1");
+        field=new Field("field1", DataType.getArray(DataType.STRING));
+        documentType.addField(field);
+    }
+
+    public void testAddSerialization() {
+        FieldUpdate update=FieldUpdate.createAdd(field, new StringFieldValue("value1"));
+        DocumentSerializer buffer = DocumentSerializerFactory.create42();
+        update.serialize(buffer);
+
+        buffer.getBuf().rewind();
+
+        try{
+            FileOutputStream fos = new FileOutputStream("src/test/files/addfieldser.dat");
+            fos.write(buffer.getBuf().array(), 0, buffer.getBuf().remaining());
+            fos.close();
+        } catch (Exception e) {}
+
+        FieldUpdate deserializedUpdate = new FieldUpdate(DocumentDeserializerFactory.create42(new DocumentTypeManager(), buffer.getBuf()), documentType, Document.SERIALIZED_VERSION);
+        assertEquals("'field1' [add value1 1]", deserializedUpdate.toString());
+    }
+
+    public void testClearSerialization() {
+        FieldUpdate update=FieldUpdate.createClear(field);
+        DocumentSerializer buffer = DocumentSerializerFactory.create42();
+        update.serialize(buffer);
+
+        buffer.getBuf().rewind();
+        FieldUpdate deserializedUpdate = new FieldUpdate(DocumentDeserializerFactory.create42(new DocumentTypeManager(), buffer.getBuf()), documentType, Document.SERIALIZED_VERSION);
+
+        assertEquals("'field1' [clear]", deserializedUpdate.toString());
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/document/update/ValueUpdateTestCase.java b/document/src/test/java/com/yahoo/document/update/ValueUpdateTestCase.java
new file mode 100644
index 00000000000..d197f77669c
--- /dev/null
+++ b/document/src/test/java/com/yahoo/document/update/ValueUpdateTestCase.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.document.update;
+
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+
+/**
+ * Test case for ValueUpdate class.
+ *
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class ValueUpdateTestCase extends junit.framework.TestCase {
+    public void testUpdateSimple() {
+        /** We cannot test much on this level anyway, most stuff in ValueUpdate is package
+         * private. Better tests exist in FieldUpdateTestCase. */
+        AssignValueUpdate upd = (AssignValueUpdate) ValueUpdate.createAssign(new StringFieldValue("newvalue"));
+
+        assertEquals(ValueUpdate.ValueUpdateClassID.ASSIGN, upd.getValueUpdateClassID());
+
+        FieldValue newValue = upd.getValue();
+        assertEquals(new StringFieldValue("newvalue"), newValue);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/PositionParserTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/PositionParserTestCase.java
new file mode 100644
index 00000000000..a40e2a5a4a3
--- /dev/null
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/PositionParserTestCase.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.datatypes.Struct;
+import org.junit.Test;
+
+import java.util.Iterator;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class PositionParserTestCase {
+
+    @Test
+    public void requireThatPositionStringsCanBeParsed() throws Exception {
+        DocumentTypeManager mgr = new DocumentTypeManager();
+        mgr.register(PositionDataType.INSTANCE);
+        DocumentType docType = new DocumentType("my_doc");
+        docType.addField("my_pos", PositionDataType.INSTANCE);
+        mgr.registerDocumentType(docType);
+
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test_position.xml", mgr);
+        Iterator<VespaXMLFeedReader.Operation> it = parser.readAll().iterator();
+        assertTrue(it.hasNext());
+        assertDocument(PositionDataType.valueOf(1, 2), it.next());
+        assertTrue(it.hasNext());
+        assertDocument(PositionDataType.fromString("E3;N4"), it.next());
+        assertTrue(it.hasNext());
+        assertDocument(PositionDataType.fromString("5;6"), it.next());
+        assertTrue(it.hasNext());
+        assertDocument(PositionDataType.fromString("7;8"), it.next());
+        assertFalse(it.hasNext());
+    }
+
+    private static void assertDocument(Struct expected, VespaXMLFeedReader.Operation operation) {
+        assertNotNull(operation);
+        assertEquals(VespaXMLFeedReader.OperationType.DOCUMENT, operation.getType());
+        Document doc = operation.getDocument();
+        assertNotNull(doc);
+        assertEquals(expected, doc.getFieldValue("my_pos"));
+    }
+}
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java
new file mode 100644
index 00000000000..2b5bf0f2b3c
--- /dev/null
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java
@@ -0,0 +1,81 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.update.AddValueUpdate;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.ValueUpdate;
+import org.junit.Test;
+
+import java.util.Iterator;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class UriParserTestCase {
+
+    @Test
+    public void requireThatUriFieldsCanBeParsed() throws Exception {
+        DocumentTypeManager mgr = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("my_doc");
+        docType.addField("my_uri", DataType.URI);
+        docType.addField("my_arr", DataType.getArray(DataType.URI));
+        mgr.registerDocumentType(docType);
+
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test_uri.xml", mgr);
+        Iterator<VespaXMLFeedReader.Operation> it = parser.readAll().iterator();
+
+        Document doc = nextDocument(it);
+        assertNotNull(doc);
+        assertEquals(new StringFieldValue("scheme://host"), doc.getFieldValue("my_uri"));
+        assertNull(doc.getFieldValue("my_arr"));
+
+        assertNotNull(doc = nextDocument(it));
+        assertNull(doc.getFieldValue("my_uri"));
+        FieldValue val = doc.getFieldValue("my_arr");
+        assertNotNull(val);
+        assertTrue(val instanceof Array);
+        Array arr = (Array)val;
+        assertEquals(1, arr.size());
+        assertEquals(new StringFieldValue("scheme://host"), arr.get(0));
+
+        DocumentUpdate upd = nextUpdate(it);
+        assertNotNull(upd);
+        assertEquals(1, upd.getFieldUpdates().size());
+        FieldUpdate fieldUpd = upd.getFieldUpdate(0);
+        assertNotNull(fieldUpd);
+        assertEquals(docType.getField("my_arr"), fieldUpd.getField());
+        assertEquals(1, fieldUpd.getValueUpdates().size());
+        ValueUpdate valueUpd = fieldUpd.getValueUpdate(0);
+        assertNotNull(valueUpd);
+        assertTrue(valueUpd instanceof AddValueUpdate);
+        assertEquals(new StringFieldValue("scheme://host"), valueUpd.getValue());
+
+        assertFalse(it.hasNext());
+    }
+
+    private static Document nextDocument(Iterator<VespaXMLFeedReader.Operation> it) {
+        assertTrue(it.hasNext());
+        VespaXMLFeedReader.Operation op = it.next();
+        assertNotNull(op);
+        assertEquals(VespaXMLFeedReader.OperationType.DOCUMENT, op.getType());
+        Document doc = op.getDocument();
+        assertNotNull(doc);
+        return doc;
+    }
+
+    private static DocumentUpdate nextUpdate(Iterator<VespaXMLFeedReader.Operation> it) {
+        assertTrue(it.hasNext());
+        VespaXMLFeedReader.Operation op = it.next();
+        assertNotNull(op);
+        assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType());
+        DocumentUpdate upd = op.getDocumentUpdate();
+        assertNotNull(upd);
+        return upd;
+    }
+}
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java
new file mode 100755
index 00000000000..19f753484d3
--- /dev/null
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java
@@ -0,0 +1,894 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
+import com.yahoo.document.serialization.DeserializationException;
+import com.yahoo.document.update.AddValueUpdate;
+import com.yahoo.document.update.MapValueUpdate;
+import com.yahoo.document.update.RemoveValueUpdate;
+import com.yahoo.document.update.ValueUpdate;
+import com.yahoo.text.Utf8;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ * Simple test case for Vespa XML parser.
+ *
+ * @author sveina
+ */
+public class VespaXMLReaderTestCase {
+
+    private final DocumentTypeManager manager = new DocumentTypeManager();
+
+    //NOTE: You generally want to extend com.yahoo.vespaxmlparser.test.documentxmltests.VespaXMLParserTestCase
+    //instead  -- and do remember to update the C++ test case also
+
+    @Before
+    public void setUp() {
+        DocumentTypeManagerConfigurer.configure(manager, "file:src/test/vespaxmlparser/documentmanager2.cfg");
+    }
+
+    @Test
+    public void testMapNoKey() throws Exception {
+        try {
+            VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/testmapnokey.xml", manager);
+            parser.readAll();
+            assertTrue(false);
+        } catch (Exception e) {
+            System.out.println(e);
+        }
+    }
+
+    @Test
+    public void testMapNoValue() throws Exception {
+        try {
+            VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/testmapnovalue.xml", manager);
+            parser.readAll();
+            assertTrue(false);
+        } catch (Exception e) {
+            System.out.println(e);
+        }
+    }
+
+    @Test
+    public void testNews1() throws Exception {
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/testalltypes.xml", manager);
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+
+        assertTrue(VespaXMLFeedReader.OperationType.INVALID != op.getType());
+        Document doc = op.getDocument();
+        assertEquals(new StringFieldValue("testUrl"), doc.getFieldValue("url"));
+        assertEquals(new StringFieldValue("testTitle"), doc.getFieldValue("title"));
+        assertEquals(new IntegerFieldValue(1), doc.getFieldValue("last_downloaded"));
+        assertEquals(new LongFieldValue(2), doc.getFieldValue("value_long"));
+        assertEquals("foobar", Utf8.toString(((Raw)doc.getFieldValue("value_raw")).getByteBuffer()));
+
+        Array strArr = (Array)doc.getFieldValue("stringarr");
+        assertEquals(new StringFieldValue("stringarrItem1"), strArr.get(0));
+        assertEquals(new StringFieldValue("stringarrItem2"), strArr.get(1));
+
+        Array intArr = (Array)doc.getFieldValue("intarr");
+        assertEquals(new IntegerFieldValue(-1311224359), intArr.get(0));
+        assertEquals(new IntegerFieldValue(-1311224358), intArr.get(1));
+        assertEquals(new IntegerFieldValue(-1), intArr.get(2));
+        assertEquals(new IntegerFieldValue(-2147483648), intArr.get(3));
+
+        Array longArr = (Array)doc.getFieldValue("longarr");
+        assertEquals(new LongFieldValue(5L), longArr.get(0));
+        assertEquals(new LongFieldValue(6L), longArr.get(1));
+
+        Array byteArr = (Array)doc.getFieldValue("bytearr");
+        assertEquals(new ByteFieldValue(7), byteArr.get(0));
+        assertEquals(new ByteFieldValue(8), byteArr.get(1));
+
+        Array floatArr = (Array)doc.getFieldValue("floatarr");
+        assertEquals(new FloatFieldValue(9.0f), floatArr.get(0));
+        assertEquals(new FloatFieldValue(10.0f), floatArr.get(1));
+
+        WeightedSet intWset = (WeightedSet)doc.getFieldValue("weightedsetint");
+        assertEquals(new Integer(11), intWset.get(new IntegerFieldValue(11)));
+        assertEquals(new Integer(12), intWset.get(new IntegerFieldValue(12)));
+
+        WeightedSet strWset = (WeightedSet)doc.getFieldValue("weightedsetstring");
+        assertEquals(new Integer(13), strWset.get(new StringFieldValue("string13")));
+        assertEquals(new Integer(14), strWset.get(new StringFieldValue("string14")));
+
+        MapFieldValue strMap = (MapFieldValue)doc.getFieldValue("stringmap");
+        assertEquals(new StringFieldValue("slovakia"), strMap.get(new StringFieldValue("italia")));
+        assertEquals(new StringFieldValue("japan"), strMap.get(new StringFieldValue("danmark")));
+        assertEquals(new StringFieldValue("new zealand"), strMap.get(new StringFieldValue("paraguay")));
+
+        Struct struct = (Struct)doc.getFieldValue("structfield");
+        assertEquals(new StringFieldValue("star wars"), struct.getFieldValue("title"));
+        assertEquals(new StringFieldValue("dummy"), struct.getFieldValue("structfield"));
+
+        List structArr = (List)doc.getFieldValue("structarr");
+        assertEquals(2, structArr.size());
+        assertEquals(new StringFieldValue("title1"), ((Struct)structArr.get(0)).getFieldValue("title"));
+        assertEquals(new StringFieldValue("title2"), ((Struct)structArr.get(1)).getFieldValue("title"));
+        assertEquals(new StringFieldValue("value1"),
+                     ((MapFieldValue)((Struct)structArr.get(0)).getFieldValue("mymap")).get(
+                             new StringFieldValue("key1")));
+        assertEquals(new StringFieldValue("value2"),
+                     ((MapFieldValue)((Struct)structArr.get(0)).getFieldValue("mymap")).get(
+                             new StringFieldValue("key2")));
+        assertEquals(new StringFieldValue("value1.1"),
+                     ((MapFieldValue)((Struct)structArr.get(1)).getFieldValue("mymap")).get(
+                             new StringFieldValue("key1.1")));
+        assertEquals(new StringFieldValue("value1.2"),
+                     ((MapFieldValue)((Struct)structArr.get(1)).getFieldValue("mymap")).get(
+                             new StringFieldValue("key1.2")));
+
+        MapFieldValue arrMap = (MapFieldValue)doc.getFieldValue("arrmap");
+        assertEquals(2, arrMap.size());
+        Array arr = (Array)arrMap.get(new StringFieldValue("foo"));
+        assertEquals(3, arr.size());
+        assertEquals(new StringFieldValue("hei1"), arr.get(0));
+        assertEquals(new StringFieldValue("hei2"), arr.get(1));
+        assertEquals(new StringFieldValue("hei3"), arr.get(2));
+        arr = (Array)arrMap.get(new StringFieldValue("bar"));
+        assertEquals(3, arr.size());
+        assertEquals(new StringFieldValue("hei4"), arr.get(0));
+        assertEquals(new StringFieldValue("hei5"), arr.get(1));
+        assertEquals(new StringFieldValue("hei6"), arr.get(2));
+    }
+
+    @Test
+    public void testNews3() throws Exception {
+        // Updating all elements in a documentType
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test03.xml", manager);
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+
+        assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType());
+
+        DocumentUpdate docUpdate = op.getDocumentUpdate();
+
+        //url
+        assertEquals(new StringFieldValue("assignUrl"), docUpdate.getFieldUpdate("url").getValueUpdate(0).getValue());
+
+        //title
+        assertEquals(new StringFieldValue("assignTitle"),
+                     docUpdate.getFieldUpdate("title").getValueUpdate(0).getValue());
+
+        //last_downloaded
+        assertEquals(new IntegerFieldValue(1),
+                     docUpdate.getFieldUpdate("last_downloaded").getValueUpdate(0).getValue());
+
+        //value_long
+        assertEquals(new LongFieldValue((long)2), docUpdate.getFieldUpdate("value_long").getValueUpdate(0).getValue());
+
+        //stringarr
+        List stringarr = (List)docUpdate.getFieldUpdate("stringarr").getValueUpdate(0).getValue();
+        assertEquals(new StringFieldValue("assignString1"), stringarr.get(0));
+        assertEquals(new StringFieldValue("assignString2"), stringarr.get(1));
+
+        //intarr
+        List intarr = (List)docUpdate.getFieldUpdate("intarr").getValueUpdate(0).getValue();
+        assertEquals(new IntegerFieldValue(3), intarr.get(0));
+        assertEquals(new IntegerFieldValue(4), intarr.get(1));
+
+        //longarr
+        List longarr = (List)docUpdate.getFieldUpdate("longarr").getValueUpdate(0).getValue();
+        assertEquals(new LongFieldValue((long)5), longarr.get(0));
+        assertEquals(new LongFieldValue((long)6), longarr.get(1));
+
+        //bytearr
+        List bytearr = (List)docUpdate.getFieldUpdate("bytearr").getValueUpdate(0).getValue();
+        assertEquals(new ByteFieldValue((byte)7), bytearr.get(0));
+        assertEquals(new ByteFieldValue((byte)8), bytearr.get(1));
+
+        //floatarr
+        List floatarr = (List)docUpdate.getFieldUpdate("floatarr").getValueUpdate(0).getValue();
+        assertEquals(new FloatFieldValue((float)9), floatarr.get(0));
+        assertEquals(new FloatFieldValue((float)10), floatarr.get(1));
+
+        //weightedsetint
+        WeightedSet weightedsetint =
+                (WeightedSet)docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(0).getValue();
+        assertEquals(new Integer(11), weightedsetint.get(new IntegerFieldValue(11)));
+        assertEquals(new Integer(12), weightedsetint.get(new IntegerFieldValue(12)));
+
+        //weightedsetstring
+        WeightedSet weightedsetstring =
+                (WeightedSet)docUpdate.getFieldUpdate("weightedsetstring").getValueUpdate(0).getValue();
+        assertEquals(new Integer(13), weightedsetstring.get(new StringFieldValue("assign13")));
+        assertEquals(new Integer(14), weightedsetstring.get(new StringFieldValue("assign14")));
+
+    }
+
+    @Test
+    public void testNews4() throws Exception {
+        // Test on adding just a few fields to a DocumentUpdate (implies other fields to null)
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test04.xml", manager);
+
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+
+        assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType());
+
+        DocumentUpdate docUpdate = op.getDocumentUpdate();
+        //url
+        assertEquals(new StringFieldValue("assignUrl"), docUpdate.getFieldUpdate("url").getValueUpdate(0).getValue());
+
+        //title
+        assertNull(docUpdate.getFieldUpdate("title"));
+
+        //last_downloaded
+        assertNull(docUpdate.getFieldUpdate("last_downloaded"));
+
+        //value_long
+        assertEquals(new LongFieldValue((long)2), docUpdate.getFieldUpdate("value_long").getValueUpdate(0).getValue());
+
+        //value_content
+        assertNull(docUpdate.getFieldUpdate("value_content"));
+
+        //stringarr
+        List stringarr = (List)docUpdate.getFieldUpdate("stringarr").getValueUpdate(0).getValue();
+        assertEquals(new StringFieldValue("assignString1"), stringarr.get(0));
+        assertEquals(new StringFieldValue("assignString2"), stringarr.get(1));
+
+        //intarr
+        List intarr = (List)docUpdate.getFieldUpdate("intarr").getValueUpdate(0).getValue();
+        assertEquals(new IntegerFieldValue(3), intarr.get(0));
+        assertEquals(new IntegerFieldValue(4), intarr.get(1));
+
+        //longarr
+        assertNull(docUpdate.getFieldUpdate("longarr"));
+
+        //bytearr
+        assertNull(docUpdate.getFieldUpdate("bytearr"));
+
+        //floatarr
+        assertNull(docUpdate.getFieldUpdate("floatarr"));
+
+        //weightedsetint
+        WeightedSet weightedsetint =
+                (WeightedSet)docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(0).getValue();
+        assertEquals(new Integer(11), weightedsetint.get(new IntegerFieldValue(11)));
+        assertEquals(new Integer(12), weightedsetint.get(new IntegerFieldValue(12)));
+
+        //weightedsetstring
+        assertNull(docUpdate.getFieldUpdate("weightedsetstring"));
+    }
+
+    @Test
+    public void testNews5() throws Exception {
+        // Adding a few new fields to a Document using different syntax
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test05.xml", manager);
+
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+
+        assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType());
+
+        DocumentUpdate docUpdate = op.getDocumentUpdate();
+
+        //url
+        assertNull(docUpdate.getFieldUpdate("url"));
+
+        //title
+        assertNull(docUpdate.getFieldUpdate("title"));
+
+        //last_downloaded
+        assertNull(docUpdate.getFieldUpdate("last_downloaded"));
+
+        //value_long
+        //assertNull(docUpdate.getFieldUpdate("value_long"));
+
+        //value_content
+        assertNull(docUpdate.getFieldUpdate("value_content"));
+
+        //stringarr
+        List stringarr = docUpdate.getFieldUpdate("stringarr").getValueUpdates();//.getValueUpdate(0).getValue();
+        assertEquals(new StringFieldValue("addString1"), ((ValueUpdate)stringarr.get(0)).getValue());
+        assertEquals(new StringFieldValue("addString2"), ((ValueUpdate)stringarr.get(1)).getValue());
+
+        //intarr
+        assertNull(docUpdate.getFieldUpdate("intarr"));
+
+        //longarr
+        List longarr = docUpdate.getFieldUpdate("longarr").getValueUpdates();
+        assertEquals(new LongFieldValue((long)5), ((ValueUpdate)longarr.get(0)).getValue());
+
+        //bytearr
+        assertNull(docUpdate.getFieldUpdate("bytearr"));
+
+        //floatarr
+        assertNull(docUpdate.getFieldUpdate("floatarr"));
+
+        //weightedsetint
+        assertEquals(new IntegerFieldValue(11), docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(0).getValue());
+        assertEquals(new Integer(11),
+                     (Integer)((AddValueUpdate)docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(0))
+                             .getWeight());
+        assertEquals(new IntegerFieldValue(12), docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(1).getValue());
+        assertEquals(new Integer(12),
+                     (Integer)((AddValueUpdate)docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(1))
+                             .getWeight());
+
+        //weightedsetstring
+        assertEquals(new StringFieldValue("add13"), docUpdate.getFieldUpdate("weightedsetstring").getValueUpdate(0).getValue());
+        assertEquals(new Integer(1),
+                     (Integer)((AddValueUpdate)docUpdate.getFieldUpdate("weightedsetstring").getValueUpdate(0))
+                             .getWeight());
+    }
+
+    @Test
+    public void testNews6() throws Exception {
+        // A document containing fields with invalid values. Different variants are used All of the updates specified in
+        // XML-file should be skipped and not added to queue Except the three of them
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test06.xml", manager);
+
+        // long value with txt
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // empty string
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+        assertEquals("doc:news:http://news6b", op.getDocument().getId().toString());
+
+        // int array with text
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // long array with whitespace
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // byte array with value
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // float array with string
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // weighted set of int with string
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // weighted set of int with string as weight
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // weighted set of string with string as weight
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        parser.read(op = new VespaXMLFeedReader.Operation());
+        assertEquals("doc:news:http://news6j", op.getDocument().getId().toString());
+
+        parser.read(op = new VespaXMLFeedReader.Operation());
+        assertEquals(VespaXMLFeedReader.OperationType.INVALID, op.getType());
+    }
+
+    @Test
+    public void testNews7() throws Exception {
+        // Testing different variants of increment/decrement/multiply/divide among with alterupdate. In test07.xml there
+        // are also some updates that will fail (be skipped).
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test07.xml", manager);
+
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+
+        assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType());
+
+        DocumentUpdate docUpdate = op.getDocumentUpdate();
+
+        List<ValueUpdate> vuList = docUpdate.getFieldUpdate("last_downloaded").getValueUpdates();
+        assertEquals(new DoubleFieldValue(2.0), vuList.get(0).getValue());
+        assertEquals(new DoubleFieldValue(3.0), vuList.get(1).getValue());
+        assertEquals(new DoubleFieldValue(4.0), vuList.get(2).getValue());
+        assertEquals(new DoubleFieldValue(5.0), vuList.get(3).getValue());
+
+        assertEquals(new IntegerFieldValue(7), docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(0).getValue());
+        assertEquals(new DoubleFieldValue(6.0), ((MapValueUpdate)docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(0))
+                .getUpdate().getValue());
+
+        assertEquals(new IntegerFieldValue(9), docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(1).getValue());
+        assertEquals(new DoubleFieldValue(8.0), ((MapValueUpdate)docUpdate.getFieldUpdate("weightedsetint").getValueUpdate(1))
+                .getUpdate().getValue());
+
+        assertEquals(new IntegerFieldValue(11), docUpdate.getFieldUpdate("intarr").getValueUpdate(0).getValue());
+        assertEquals(new DoubleFieldValue(10.0),
+                     ((MapValueUpdate)docUpdate.getFieldUpdate("intarr").getValueUpdate(0)).getUpdate().getValue());
+
+        assertEquals(new IntegerFieldValue(13), docUpdate.getFieldUpdate("floatarr").getValueUpdate(0).getValue());
+        assertEquals(new DoubleFieldValue(12.0),
+                     ((MapValueUpdate)docUpdate.getFieldUpdate("floatarr").getValueUpdate(0)).getUpdate().getValue());
+
+        assertEquals(new IntegerFieldValue(15), docUpdate.getFieldUpdate("floatarr").getValueUpdate(1).getValue());
+        assertEquals(new DoubleFieldValue(14.0),
+                     ((MapValueUpdate)docUpdate.getFieldUpdate("floatarr").getValueUpdate(1)).getUpdate().getValue());
+
+        // Trying arithmetic on string (b)
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // "By" as string (c)
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // Empty key in weighted set of int (d)
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // No "by" attribute (e)
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+
+        // Float key as string (f)
+        try {
+            parser.read(new VespaXMLFeedReader.Operation());
+            fail();
+        } catch (Exception e) {
+            System.out.println(e.getMessage());
+        }
+    }
+
+    @Test
+    public void testNews8() throws Exception {
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test08.xml", manager);
+
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+
+        assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType());
+
+        DocumentUpdate docUpdate = op.getDocumentUpdate();
+
+        //stringarr
+        List<ValueUpdate> vuList = docUpdate.getFieldUpdate("stringarr").getValueUpdates();
+        assertTrue(vuList.get(0) instanceof RemoveValueUpdate);
+        assertEquals(new StringFieldValue("removeString1"), vuList.get(0).getValue());
+        assertTrue(vuList.get(1) instanceof RemoveValueUpdate);
+        assertEquals(new StringFieldValue("removeString2"), vuList.get(1).getValue());
+
+        //weightedsetint
+        vuList = docUpdate.getFieldUpdate("weightedsetint").getValueUpdates();
+        assertEquals(2, vuList.size());
+        assertTrue(vuList.contains(new RemoveValueUpdate(new IntegerFieldValue(5))));
+        assertTrue(vuList.contains(new RemoveValueUpdate(new IntegerFieldValue(4))));
+    }
+
+    @Test
+    public void testNews9() throws Exception {
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test09.xml", manager);
+
+        {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+
+            assertEquals(VespaXMLFeedReader.OperationType.REMOVE, op.getType());
+            assertEquals("doc:news:http://news9a", op.getRemove().toString());
+        }
+        {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+
+            assertEquals(VespaXMLFeedReader.OperationType.REMOVE, op.getType());
+            assertEquals("doc:news:http://news9b", op.getRemove().toString());
+        }
+        {
+            // Remove without documentid. Not supported.
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            try {
+                parser.read(op);
+                fail();
+            } catch (Exception e) {
+                System.out.println(e.getMessage());
+            }
+        }
+    }
+
+    @Test
+    public void testNews10() throws Exception {
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test10.xml", manager);
+        {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+            Document doc = op.getDocument();
+
+            assertEquals(new StringFieldValue("testUrl"), doc.getFieldValue("url"));
+            assertEquals(new StringFieldValue("testTitle"), doc.getFieldValue("title"));
+            assertEquals(new IntegerFieldValue(1), doc.getFieldValue("last_downloaded"));
+            assertEquals(new LongFieldValue(2), doc.getFieldValue("value_long"));
+
+            Array strArr = (Array)doc.getFieldValue("stringarr");
+            assertEquals(new StringFieldValue("stringarrItem1"), strArr.get(0));
+            assertEquals(new StringFieldValue("stringarrItem2"), strArr.get(1));
+
+            Array intArr = (Array)doc.getFieldValue("intarr");
+            assertEquals(new IntegerFieldValue(3), intArr.get(0));
+            assertEquals(new IntegerFieldValue(4), intArr.get(1));
+
+            Array longArr = (Array)doc.getFieldValue("longarr");
+            assertEquals(new LongFieldValue(5L), longArr.get(0));
+            assertEquals(new LongFieldValue(6L), longArr.get(1));
+
+            Array byteArr = (Array)doc.getFieldValue("bytearr");
+            assertEquals(new ByteFieldValue((byte)7), byteArr.get(0));
+            assertEquals(new ByteFieldValue((byte)8), byteArr.get(1));
+
+            Array floatArr = (Array)doc.getFieldValue("floatarr");
+            assertEquals(new FloatFieldValue(9.0f), floatArr.get(0));
+            assertEquals(new FloatFieldValue(10.0f), floatArr.get(1));
+
+            WeightedSet intWset = (WeightedSet)doc.getFieldValue("weightedsetint");
+            assertEquals(new Integer(11), intWset.get(new IntegerFieldValue(11)));
+            assertEquals(new Integer(12), intWset.get(new IntegerFieldValue(12)));
+
+            WeightedSet strWset = (WeightedSet)doc.getFieldValue("weightedsetstring");
+            assertEquals(new Integer(13), strWset.get(new StringFieldValue("string13")));
+            assertEquals(new Integer(14), strWset.get(new StringFieldValue("string14")));
+        }
+        {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+            Document doc = op.getDocument();
+            assertNotNull(doc);
+            assertEquals(new StringFieldValue("testUrl2"), doc.getFieldValue("url"));
+        }
+        {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+            DocumentUpdate upd = op.getDocumentUpdate();
+
+            assertNull(upd.getFieldUpdate("url"));
+            assertNull(upd.getFieldUpdate("title"));
+            assertNull(upd.getFieldUpdate("last_downloaded"));
+            assertNull(upd.getFieldUpdate("value_long"));
+            assertNull(upd.getFieldUpdate("value_content"));
+
+            List<ValueUpdate> lst = upd.getFieldUpdate("stringarr").getValueUpdates();
+            assertEquals(new StringFieldValue("addString1"), lst.get(0).getValue());
+            assertEquals(new StringFieldValue("addString2"), lst.get(1).getValue());
+
+            assertNull(upd.getFieldUpdate("intarr"));
+
+            lst = upd.getFieldUpdate("longarr").getValueUpdates();
+            assertEquals(new LongFieldValue((long)5), lst.get(0).getValue());
+
+            assertNull(upd.getFieldUpdate("bytearr"));
+            assertNull(upd.getFieldUpdate("floatarr"));
+
+            assertEquals(new IntegerFieldValue(11), upd.getFieldUpdate("weightedsetint").getValueUpdate(0).getValue());
+            assertEquals(new Integer(11),
+                         (Integer)((AddValueUpdate)upd.getFieldUpdate("weightedsetint").getValueUpdate(0))
+                                 .getWeight());
+            assertEquals(new IntegerFieldValue(12), upd.getFieldUpdate("weightedsetint").getValueUpdate(1).getValue());
+            assertEquals(new Integer(12),
+                         (Integer)((AddValueUpdate)upd.getFieldUpdate("weightedsetint").getValueUpdate(1))
+                                 .getWeight());
+
+            assertEquals(new StringFieldValue("add13"), upd.getFieldUpdate("weightedsetstring").getValueUpdate(0).getValue());
+            assertEquals(new Integer(1),
+                         (Integer)((AddValueUpdate)upd.getFieldUpdate("weightedsetstring").getValueUpdate(0))
+                                 .getWeight());
+        }
+        {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+            DocumentUpdate upd = op.getDocumentUpdate();
+
+            assertEquals(new StringFieldValue("assignUrl"),
+                         upd.getFieldUpdate("url").getValueUpdate(0).getValue());
+
+            assertNull(upd.getFieldUpdate("title"));
+            assertNull(upd.getFieldUpdate("last_downloaded"));
+            assertEquals(new LongFieldValue(2),
+                         upd.getFieldUpdate("value_long").getValueUpdate(0).getValue());
+            assertNull(upd.getFieldUpdate("value_content"));
+
+            Array strArr = (Array)upd.getFieldUpdate("stringarr").getValueUpdate(0).getValue();
+            assertEquals(new StringFieldValue("assignString1"), strArr.get(0));
+            assertEquals(new StringFieldValue("assignString2"), strArr.get(1));
+
+            Array intArr = (Array)upd.getFieldUpdate("intarr").getValueUpdate(0).getValue();
+            assertEquals(new IntegerFieldValue(3), intArr.get(0));
+            assertEquals(new IntegerFieldValue(4), intArr.get(1));
+
+            assertNull(upd.getFieldUpdate("longarr"));
+            assertNull(upd.getFieldUpdate("bytearr"));
+            assertNull(upd.getFieldUpdate("floatarr"));
+
+            WeightedSet intWset = (WeightedSet)upd.getFieldUpdate("weightedsetint").getValueUpdate(0).getValue();
+            assertEquals(new Integer(11), intWset.get(new IntegerFieldValue(11)));
+            assertEquals(new Integer(12), intWset.get(new IntegerFieldValue(12)));
+
+            assertNull(upd.getFieldUpdate("weightedsetstring"));
+        }
+        {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+            assertEquals("doc:news:http://news10e", op.getRemove().toString());
+        }
+        {
+            // Illegal remove without documentid attribute
+            try {
+                VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+                parser.read(op);
+                fail();
+            } catch (Exception e) {
+                System.out.println(e.getMessage());
+            }
+        }
+    }
+
+    @Test
+    public void testFieldPathUpdates() throws Exception {
+        // Adding a few new fields to a Document using different syntax
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/tests/vespaxml/fieldpathupdates.xml", manager);
+
+        VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+        parser.read(op);
+
+        assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType());
+
+        DocumentUpdate docUpdate = op.getDocumentUpdate();
+
+        assertEquals(20, docUpdate.getFieldPathUpdates().size());
+
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(0);
+            assertEquals("url", ass.getOriginalFieldPath());
+            assertEquals(new StringFieldValue("assignUrl"), ass.getNewValue());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(1);
+            assertEquals("title", ass.getOriginalFieldPath());
+            assertEquals(new StringFieldValue("assignTitle"), ass.getNewValue());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(2);
+            assertEquals("last_downloaded", ass.getOriginalFieldPath());
+            assertEquals("1", ass.getExpression());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(3);
+            assertEquals("value_long", ass.getOriginalFieldPath());
+            assertEquals("2", ass.getExpression());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(5);
+            assertEquals("stringarr", ass.getOriginalFieldPath());
+            assertEquals("[assignString1, assignString2]", ass.getNewValue().toString());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(6);
+            assertEquals("intarr", ass.getOriginalFieldPath());
+            assertEquals("[3, 4]", ass.getNewValue().toString());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(7);
+            assertEquals("longarr", ass.getOriginalFieldPath());
+            assertEquals("[5, 6]", ass.getNewValue().toString());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(8);
+            assertEquals("bytearr", ass.getOriginalFieldPath());
+            assertEquals("[7, 8]", ass.getNewValue().toString());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(9);
+            assertEquals("floatarr", ass.getOriginalFieldPath());
+            assertEquals("[9.0, 10.0]", ass.getNewValue().toString());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(10);
+            assertEquals("weightedsetint", ass.getOriginalFieldPath());
+            WeightedSet set = (WeightedSet)ass.getNewValue();
+            assertEquals(new Integer(11), set.get(new IntegerFieldValue(11)));
+            assertEquals(new Integer(12), set.get(new IntegerFieldValue(12)));
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(11);
+            assertEquals("weightedsetstring", ass.getOriginalFieldPath());
+            WeightedSet set = (WeightedSet)ass.getNewValue();
+            assertEquals(new Integer(13), set.get(new StringFieldValue("assign13")));
+            assertEquals(new Integer(14), set.get(new StringFieldValue("assign14")));
+        }
+        {
+            AddFieldPathUpdate ass = (AddFieldPathUpdate)docUpdate.getFieldPathUpdates().get(12);
+            assertEquals("stringarr", ass.getOriginalFieldPath());
+            assertEquals("[addString1, addString2]", ass.getNewValues().toString());
+        }
+        {
+            AddFieldPathUpdate ass = (AddFieldPathUpdate)docUpdate.getFieldPathUpdates().get(13);
+            assertEquals("longarr", ass.getOriginalFieldPath());
+            assertEquals("[5]", ass.getNewValues().toString());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(14);
+            assertEquals("weightedsetint{13}", ass.getOriginalFieldPath());
+            assertEquals("13", ass.getExpression());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(15);
+            assertEquals("weightedsetint{14}", ass.getOriginalFieldPath());
+            assertEquals("14", ass.getExpression());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(16);
+            assertEquals("weightedsetstring{add13}", ass.getOriginalFieldPath());
+            assertEquals("1", ass.getExpression());
+        }
+        {
+            AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(17);
+            assertEquals("weightedsetstring{assign13}", ass.getOriginalFieldPath());
+            assertEquals("130", ass.getExpression());
+        }
+        {
+            RemoveFieldPathUpdate ass = (RemoveFieldPathUpdate)docUpdate.getFieldPathUpdates().get(18);
+            assertEquals("weightedsetstring{assign14}", ass.getOriginalFieldPath());
+        }
+        {
+            RemoveFieldPathUpdate ass = (RemoveFieldPathUpdate)docUpdate.getFieldPathUpdates().get(19);
+            assertEquals("bytearr", ass.getOriginalFieldPath());
+        }
+        Document doc = new Document(manager.getDocumentType("news"), new DocumentId("doc:test:test:test"));
+        docUpdate.applyTo(doc);
+    }
+
+    @Test
+    public void testDocInDoc() throws Exception {
+        DocumentTypeManager m = new DocumentTypeManager();
+        DocumentTypeManagerConfigurer
+                .configure(m, "file:src/test/java/com/yahoo/document/documentmanager.docindoc.cfg");
+
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test_docindoc.xml", m);
+        List<VespaXMLFeedReader.Operation> ops = parser.readAll();
+
+        assertEquals(1, ops.size());
+        VespaXMLFeedReader.Operation op = ops.get(0);
+        System.err.println(op);
+
+        assertEquals(VespaXMLFeedReader.OperationType.DOCUMENT, op.getType());
+        assertNull(op.getRemove());
+        assertNull(op.getDocumentUpdate());
+        assertNull(op.getFeedOperation());
+        assertNotNull(op.getDocument());
+
+        Document doc = op.getDocument();
+
+        assertEquals("outerdoc", doc.getDataType().getName());
+        assertEquals("doc:outer:this:is:outer:doc", doc.getId().toString());
+        assertEquals(1, doc.getFieldCount());
+
+        Array lst = (Array)doc.getFieldValue("innerdocuments");
+        assertNotNull(lst);
+        assertEquals(3, lst.size());
+
+        Document child = (Document)lst.get(0);
+        assertEquals(2, child.getFieldCount());
+        assertEquals("Peter Sellers", child.getFieldValue("name").toString());
+        assertEquals("Comedian", child.getFieldValue("content").toString());
+
+        child = (Document)lst.get(1);
+        assertEquals(2, child.getFieldCount());
+        assertEquals("Ole Olsen", child.getFieldValue("name").toString());
+        assertEquals("Common man", child.getFieldValue("content").toString());
+
+        child = (Document)lst.get(2);
+        assertEquals(2, child.getFieldCount());
+        assertEquals("Stein Nilsen", child.getFieldValue("name").toString());
+        assertEquals("Worker", child.getFieldValue("content").toString());
+    }
+
+    @Test(expected = DeserializationException.class)
+    public void testBinaryEncodingStrings() throws Exception {
+        DocumentTypeManager dtm = new DocumentTypeManager();
+
+        DocumentType type = new DocumentType("foo");
+        type.addField(new Field("title", DataType.STRING));
+
+        dtm.registerDocumentType(type);
+
+        String input =
+                "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" +
+                        "<vespafeed>\n" +
+                        "    <document documenttype=\"foo\" documentid=\"doc:foo:bar:baz\"> \n" +
+                        "        <title binaryencoding=\"base64\">testTitle</title>\n" +
+                        "    </document>\n" +
+                        "</vespafeed>\n";
+
+        VespaXMLFeedReader parser = new VespaXMLFeedReader(new ByteArrayInputStream(Utf8.toBytes(input)), dtm);
+        parser.readAll();
+    }
+
+    @Test(expected = DeserializationException.class)
+    public void testIllegalCharacterInStrings() throws Exception {
+        DocumentTypeManager dtm = new DocumentTypeManager();
+
+        DocumentType type = new DocumentType("foo");
+        type.addField(new Field("title", DataType.STRING));
+
+        dtm.registerDocumentType(type);
+
+        String input =
+                "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" +
+                        "<vespafeed>\n" +
+                        "    <document documenttype=\"foo\" documentid=\"doc:foo:bar:baz\"> \n" +
+                        "        <title>test\uFDDFTitle</title>\n" +
+                        "    </document>\n" +
+                        "</vespafeed>\n";
+
+        VespaXMLFeedReader parser = new VespaXMLFeedReader(new ByteArrayInputStream(Utf8.toBytes(input)), dtm);
+        parser.readAll();
+    }
+
+    @Test
+    public void testTestAndSetConditionAttribute() throws Exception {
+        VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/testandset.xml", manager);
+        final int NUM_OPERATIONS_IN_FEED = 3;
+
+        for (int i = 0; i < NUM_OPERATIONS_IN_FEED; i++) {
+            VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation();
+            parser.read(op);
+
+            assertTrue("Missing test and set condition", op.getCondition().isPresent());
+            assertEquals("Condition is not the same as in xml feed", "news.value_long == 1", op.getCondition().getSelection());
+        }
+    }
+
+}
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java
new file mode 100644
index 00000000000..3cfbcac5b62
--- /dev/null
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlFieldReaderTestCase.java
@@ -0,0 +1,181 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.predicate.BinaryFormat;
+import com.yahoo.document.predicate.Conjunction;
+import com.yahoo.document.predicate.FeatureRange;
+import com.yahoo.document.predicate.FeatureSet;
+import com.yahoo.document.predicate.Predicate;
+import com.yahoo.document.serialization.DeserializationException;
+import org.apache.commons.codec.binary.Base64;
+import org.junit.Test;
+
+import javax.xml.stream.XMLStreamReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class VespaXmlFieldReaderTestCase {
+
+    @Test
+    public void requireThatPredicateFieldValuesCanBeRead() throws Exception {
+        assertReadable(new Conjunction(new FeatureSet("foo", "bar"),
+                                       new FeatureRange("baz", 6L, 9L)));
+    }
+
+    @Test
+    public void requireThatArrayItemDeserializeExceptionIncludesFieldName() throws Exception {
+        assertThrows(new Field("my_field", DataType.getArray(DataType.BYTE)),
+                     "<item>-129</item>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 70)");
+    }
+
+    @Test
+    public void requireThatMapKeyDeserializeExceptionIncludesFieldName() throws Exception {
+        assertThrows(new Field("my_field", DataType.getMap(DataType.BYTE, DataType.STRING)),
+                     "<item><key>-129</key><value>foo</value></item>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 74)");
+    }
+
+    @Test
+    public void requireThatMapValueDeserializeExceptionIncludesFieldName() throws Exception {
+        assertThrows(new Field("my_field", DataType.getMap(DataType.STRING, DataType.BYTE)),
+                     "<item><key>foo</key><value>-129</value></item>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 92)");
+    }
+
+    @Test
+    public void requireThatStructFieldDeserializeExceptionIncludesFieldName() throws Exception {
+        StructDataType structType = new StructDataType("my_struct");
+        structType.addField(new Field("my_byte", DataType.BYTE));
+        assertThrows(new Field("my_field", structType),
+                     "<my_byte>-129</my_byte>",
+                     "Field 'my_byte': Invalid byte \"-129\". (at line 1, column 76)");
+    }
+
+    @Test
+    public void requireThatWSetItemDeserializeExceptionIncludesFieldName() throws Exception {
+        assertThrows(new Field("my_field", DataType.getWeightedSet(DataType.BYTE)),
+                     "<item>-129</item>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 70)");
+    }
+
+    @Test
+    public void requireThatPutsForTensorFieldsAreNotSupported() throws Exception {
+        assertThrows(new Field("my_tensor", DataType.TENSOR), "",
+                "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead.");
+    }
+
+    private class MockedReaderFixture {
+        public DocumentTypeManager mgr;
+        public DocumentType docType;
+        public XMLStreamReader xmlReader;
+        public VespaXMLFieldReader fieldReader;
+
+        public MockedReaderFixture() {
+            mgr = new DocumentTypeManager();
+            mgr.register(PositionDataType.INSTANCE);
+            docType = new DocumentType("my_doc");
+            docType.addField("my_pos", PositionDataType.INSTANCE);
+            mgr.registerDocumentType(docType);
+
+            xmlReader = mock(XMLStreamReader.class);
+            fieldReader = new VespaXMLFieldReader(xmlReader, mgr);
+        }
+
+        public void assertReadPositionEquals(int x, int y) {
+            Struct pos = new Struct(PositionDataType.INSTANCE);
+            fieldReader.read(docType.getField("my_pos"), pos);
+
+            assertEquals(new IntegerFieldValue(x), pos.getFieldValue(PositionDataType.FIELD_X));
+            assertEquals(new IntegerFieldValue(y), pos.getFieldValue(PositionDataType.FIELD_Y));
+        }
+    }
+
+    @Test
+    public void requireThatPositionFieldCanBeReadInSingleEvent() throws Exception {
+        MockedReaderFixture fixture = new MockedReaderFixture();
+        XMLStreamReader xmlReader = fixture.xmlReader;
+
+        when(xmlReader.getAttributeCount()).thenReturn(0);
+        when(xmlReader.hasNext()).thenReturn(true, true, false);
+        when(xmlReader.next()).thenReturn(
+                XMLStreamReader.CHARACTERS, XMLStreamReader.END_ELEMENT);
+        when(xmlReader.getText()).thenReturn("E3;N4");
+
+        fixture.assertReadPositionEquals(3000000, 4000000);
+    }
+
+    @Test
+    public void requireThatPositionFieldCanBeReadAcrossMultipleEvents() throws Exception {
+        MockedReaderFixture fixture = new MockedReaderFixture();
+        XMLStreamReader xmlReader = fixture.xmlReader;
+
+        when(xmlReader.getAttributeCount()).thenReturn(0);
+        when(xmlReader.hasNext()).thenReturn(true, true, true, true, false);
+        when(xmlReader.next()).thenReturn(
+                XMLStreamReader.CHARACTERS, XMLStreamReader.CHARACTERS,
+                XMLStreamReader.CHARACTERS, XMLStreamReader.END_ELEMENT);
+        when(xmlReader.getText()).thenReturn("E3;", "N", "4");
+
+        fixture.assertReadPositionEquals(3000000, 4000000);
+    }
+
+    private static void assertThrows(Field field, String fieldXml, String expected) throws Exception {
+        DocumentTypeManager docManager = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField(field);
+        docManager.register(docType);
+
+        String documentXml = "<document id='doc:scheme:' type='my_type'><" + field.getName() + ">" +
+                             fieldXml + "</" + field.getName() + "></document>";
+        InputStream in = new ByteArrayInputStream(documentXml.getBytes(StandardCharsets.UTF_8));
+        Document doc = new Document(docType, "doc:scheme:");
+        try {
+            new VespaXMLFieldReader(in, docManager).read(null, doc);
+            fail();
+        } catch (DeserializationException e) {
+            assertEquals(expected, e.getMessage());
+        }
+    }
+
+    private static void assertReadable(Predicate predicate) throws Exception {
+        assertRead(predicate,
+                   "<document id='doc:scheme:' type='my_type'>" +
+                   "  <my_predicate>" + predicate + "</my_predicate>" +
+                   "</document>");
+        assertRead(predicate,
+                   "<document id='doc:scheme:' type='my_type'>" +
+                   "  <my_predicate binaryencoding='base64'>" +
+                   Base64.encodeBase64String(BinaryFormat.encode(predicate)) +
+                   "  </my_predicate>" +
+                   "</document>");
+    }
+
+    private static void assertRead(Predicate expected, String documentXml) throws Exception {
+        DocumentTypeManager docManager = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField("my_predicate", DataType.PREDICATE);
+        docManager.register(docType);
+
+        InputStream in = new ByteArrayInputStream(documentXml.getBytes(StandardCharsets.UTF_8));
+        Document doc = new Document(docType, "doc:scheme:");
+        new VespaXMLFieldReader(in, docManager).read(null, doc);
+        FieldValue value = doc.getFieldValue("my_predicate");
+        assertTrue(value instanceof PredicateFieldValue);
+        assertEquals(expected, ((PredicateFieldValue)value).getPredicate());
+    }
+}
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java
new file mode 100644
index 00000000000..8730265c80d
--- /dev/null
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXmlUpdateReaderTestCase.java
@@ -0,0 +1,262 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.serialization.DeserializationException;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class VespaXmlUpdateReaderTestCase {
+
+    @Test
+    @Ignore
+    public void requireThatArithmeticDeserializationValidateValue() throws Exception {
+        // tracked in ticket 6675085
+        // problem caused by VespaXMLUpdateReader#readArithmetic() parsing value as double
+        Field field = new Field("my_field", DataType.BYTE);
+        assertThrows(field,
+                     "<increment field='my_field' by='-129' />",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column X)");
+        assertThrows(field,
+                     "<decrement field='my_field' by='-129' />",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column X)");
+        assertThrows(field,
+                     "<multiply field='my_field' by='-129' />",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column X)");
+        assertThrows(field,
+                     "<divide field='my_field' by='-129' />",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column X)");
+        assertThrows(field,
+                     "<alter field='my_field'><increment by='-129' /></alter>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column X)");
+    }
+
+    @Test
+    @Ignore
+    public void requireThatAssignNumericFieldPathValidatesFieldValue() throws Exception {
+        // tracked in ticket 6675089
+        // problem caused by VespaXMLUpdateReader#read(AssignFieldPathUpdate)
+        assertThrows(new Field("my_field", DataType.BYTE),
+                     "<assign fieldpath='my_field'>-129</assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column X)");
+    }
+
+    @Test
+    @Ignore
+    public void requireThatFieldPathWhereClauseIsValidated() throws Exception {
+        // tracked in ticket 6675091
+        // problem caused by VespaXMLUpdateReader#read(FieldPathUpdate) not validating where clause
+        assertThrows(new Field("my_field", DataType.getArray(DataType.BYTE)),
+                     "<remove fieldpath='my_field[$x]' where='my_field[$x] == -129' />",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column X)");
+        assertThrows(new Field("my_field", DataType.getMap(DataType.STRING, DataType.BYTE)),
+                     "<remove fieldpath='my_field{$x}' where='my_field{$x} == -129' />",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 109)");
+    }
+
+    @Test
+    public void requireThatDeserializeExceptionIncludesFieldName() throws Exception {
+        assertThrows(new Field("my_field", DataType.BYTE),
+                     "<assign field='my_field'>-129</assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 79)");
+    }
+
+    @Test
+    public void requireThatArrayItemDeserializeExceptionIncludesFieldName() throws Exception {
+        Field field = new Field("my_field", DataType.getArray(DataType.BYTE));
+        assertThrows(field,
+                     "<assign field='my_field'><item>-129</item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 83)");
+        assertThrows(field,
+                     "<assign fieldpath='my_field'><item>-129</item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 87)");
+        assertThrows(field,
+                     "<add field='my_field'><item>-129</item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 80)");
+        assertThrows(field,
+                     "<add fieldpath='my_field'><item>-129</item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 84)");
+        assertThrows(field,
+                     "<remove field='my_field'><item>-129</item></remove>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 83)");
+    }
+
+    @Test
+    public void requireThatMapKeyDeserializeExceptionIncludesFieldName() throws Exception {
+        Field field = new Field("my_field", DataType.getMap(DataType.BYTE, DataType.STRING));
+        assertThrows(field,
+                     "<assign field='my_field'><item><key>-129</key><value>foo</value></item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 87)");
+        assertThrows(field,
+                     "<assign fieldpath='my_field'><item><key>-129</key><value>foo</value></item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 91)");
+        assertThrows(field,
+                     "<add field='my_field'><item><key>-129</key><value>foo</value></item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 84)");
+        assertThrows(field,
+                     "<add fieldpath='my_field'><item><key>-129</key><value>foo</value></item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 88)");
+        assertThrows(field,
+                     "<remove field='my_field'><item><key>-129</key><value>foo</value></item></remove>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 87)");
+        try {
+            readUpdate(field, "<remove fieldpath='my_field{-129}' />");
+            fail();
+        } catch (NumberFormatException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatMapValueDeserializeExceptionIncludesFieldName() throws Exception {
+        Field field = new Field("my_field", DataType.getMap(DataType.STRING, DataType.BYTE));
+        assertThrows(field,
+                     "<assign field='my_field'><item><key>foo</key><value>-129</value></item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 105)");
+        assertThrows(field,
+                     "<assign fieldpath='my_field'><item><key>foo</key><value>-129</value></item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 109)");
+        assertThrows(field,
+                     "<add field='my_field'><item><key>foo</key><value>-129</value></item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 102)");
+        assertThrows(field,
+                     "<add fieldpath='my_field'><item><key>foo</key><value>-129</value></item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 106)");
+        assertThrows(field,
+                     "<remove field='my_field'><item><key>foo</key><value>-129</value></item></remove>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 105)");
+    }
+
+    @Test
+    public void requireThatStructFieldDeserializeExceptionIncludesFieldName() throws Exception {
+        StructDataType structType = new StructDataType("my_struct");
+        structType.addField(new Field("my_byte", DataType.BYTE));
+        Field field = new Field("my_field", structType);
+        assertThrows(field,
+                     "<assign field='my_field'><my_byte>-129</my_byte></assign>",
+                     "Field 'my_byte': Invalid byte \"-129\". (at line 1, column 89)");
+        assertThrows(field,
+                     "<assign fieldpath='my_field'><my_byte>-129</my_byte></assign>",
+                     "Field 'my_byte': Invalid byte \"-129\". (at line 1, column 93)");
+        assertThrows(field,
+                     "<add field='my_field'><my_byte>-129</my_byte></add>",
+                     "Field 'my_byte': Invalid byte \"-129\". (at line 1, column 86)");
+        assertThrows(field,
+                     "<add fieldpath='my_field'><my_byte>-129</my_byte></add>",
+                     "Field 'my_byte': Invalid byte \"-129\". (at line 1, column 90)");
+        assertThrows(field,
+                     "<remove field='my_field'><my_byte>-129</my_byte></remove>",
+                     "Field 'my_byte': Invalid byte \"-129\". (at line 1, column 89)");
+    }
+
+    @Test
+    public void requireThatWSetItemDeserializeExceptionIncludesFieldName() throws Exception {
+        Field field = new Field("my_field", DataType.getWeightedSet(DataType.BYTE));
+        assertThrows(field,
+                     "<assign field='my_field'><item>-129</item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 83)");
+        assertThrows(field,
+                     "<assign fieldpath='my_field'><item>-129</item></assign>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 87)");
+        assertThrows(field,
+                     "<add field='my_field'><item>-129</item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 80)");
+        assertThrows(field,
+                     "<add fieldpath='my_field'><item>-129</item></add>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 84)");
+        assertThrows(field,
+                     "<remove field='my_field'><item>-129</item></remove>",
+                     "Field 'my_field': Invalid byte \"-129\". (at line 1, column 83)");
+        try {
+            readUpdate(field, "<remove fieldpath='my_field{-129}' />");
+            fail();
+        } catch (NumberFormatException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatCreateIfNonExistentFlagCanBeSpecified() throws Exception {
+        {
+            assertTrue(readUpdate(true).getCreateIfNonExistent());
+            assertFalse(readUpdate(false).getCreateIfNonExistent());
+        }
+    }
+
+    @Test
+    public void requireThatCreateIfNonExistentFlagIsValidated() throws Exception {
+        String documentXml = "<update id='doc:scheme:' type='my_type' create-if-non-existent='illegal'></update>";
+        try {
+            readUpdateHelper(null, documentXml);
+            fail();
+        } catch (DeserializationException e) {
+            assertEquals(printStackTrace(e), "'create-if-non-existent' must be either 'true' or 'false', was 'illegal' (at line 1, column 74)", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatUpdatesForTensorFieldsAreNotSupported() throws Exception {
+        assertThrows(new Field("my_tensor", DataType.TENSOR), "<assign field='my_tensor'></assign>",
+                "Field 'my_tensor': XML input for fields of type TENSOR is not supported. Please use JSON input instead.");
+    }
+
+    private static void assertThrows(Field field, String fieldXml, String expected) throws Exception {
+        try {
+            readUpdate(field, fieldXml);
+            fail();
+        } catch (DeserializationException e) {
+            assertEquals(printStackTrace(e), expected, e.getMessage());
+        }
+    }
+
+    private static DocumentUpdate readUpdate(Field field, String fieldXml) throws Exception {
+        String documentXml = "<update id='doc:scheme:' type='my_type'>" + fieldXml + "</update>";
+        return readUpdateHelper(field, documentXml);
+    }
+
+    private static DocumentUpdate readUpdate(boolean createIfNonExistent) throws Exception {
+        String documentXml = "<update id='doc:scheme:' type='my_type' create-if-non-existent='" + (createIfNonExistent ? "true" : "false") + "'></update>";
+        return readUpdateHelper(null, documentXml);
+    }
+
+    private static DocumentUpdate readUpdateHelper(Field field, String documentXml) throws Exception {
+        DocumentTypeManager docManager = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("my_type");
+        if (field != null) {
+            docType.addField(field);
+        }
+        docManager.register(docType);
+
+        InputStream in = new ByteArrayInputStream(documentXml.getBytes(StandardCharsets.UTF_8));
+        DocumentUpdate doc = new DocumentUpdate(docType, "doc:scheme:");
+        VespaXMLUpdateReader reader = new VespaXMLUpdateReader(in, docManager);
+        reader.reader.next(); // initialize reader
+        reader.read(doc);
+        return doc;
+    }
+
+    private static String printStackTrace(Throwable t) {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        t.printStackTrace(new PrintStream(out));
+        return new String(out.toByteArray(), StandardCharsets.UTF_8);
+    }
+}
diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/XMLNumericFieldErrorMsgTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/XMLNumericFieldErrorMsgTestCase.java
new file mode 100644
index 00000000000..1f74cd650ae
--- /dev/null
+++ b/document/src/test/java/com/yahoo/vespaxmlparser/XMLNumericFieldErrorMsgTestCase.java
@@ -0,0 +1,114 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespaxmlparser;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.serialization.DeserializationException;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @since 5.1.29
+ */
+public class XMLNumericFieldErrorMsgTestCase {
+
+    private static DocumentTypeManager setupTypes() {
+        DocumentTypeManager dtm = new DocumentTypeManager();
+        DocumentType docType = new DocumentType("doctype");
+        docType.addField("bytefield", DataType.BYTE);
+        docType.addField("intfield", DataType.INT);
+        docType.addField("longfield", DataType.LONG);
+        docType.addField("floatfield", DataType.FLOAT);
+        docType.addField("doublefield", DataType.DOUBLE);
+        dtm.register(docType);
+        return dtm;
+    }
+
+    @Test
+    public void requireDescriptiveErrorMsgForFloats() throws Exception {
+        DocumentTypeManager dtm = setupTypes();
+        try {
+            VespaXMLDocumentReader documentReader = new VespaXMLDocumentReader(
+                    new ByteArrayInputStream(("<document id=\"doc:foo:bar\" type=\"doctype\">" +
+                                              "  <floatfield></floatfield>" +
+                                              "</document>").getBytes(StandardCharsets.UTF_8)), dtm);
+            new Document(documentReader);
+            fail("Sorry mac");
+        } catch (DeserializationException e) {
+            assertThat(e.getMessage(), e.getMessage().contains("Field 'floatfield': Invalid float \"\""), is(true));
+        }
+    }
+
+    @Test
+    public void requireDescriptiveErrorMsgForDoubles() throws Exception {
+        DocumentTypeManager dtm = setupTypes();
+        try {
+            VespaXMLDocumentReader documentReader = new VespaXMLDocumentReader(
+                    new ByteArrayInputStream(("<document id=\"doc:foo:bar\" type=\"doctype\">" +
+                                              "  <doublefield></doublefield>" +
+                                              "</document>").getBytes(StandardCharsets.UTF_8)), dtm);
+            new Document(documentReader);
+            fail("Sorry mac");
+        } catch (DeserializationException e) {
+            assertThat(e.getMessage(), e.getMessage().contains("Field 'doublefield': Invalid double \"\""), is(true));
+        }
+    }
+
+    @Test
+    public void requireDescriptiveErrorMsgForLongs() throws Exception {
+        DocumentTypeManager dtm = setupTypes();
+        try {
+            VespaXMLDocumentReader documentReader = new VespaXMLDocumentReader(
+                    new ByteArrayInputStream(("<document id=\"doc:foo:bar\" type=\"doctype\">" +
+                                              "  <longfield></longfield>" +
+                                              "</document>").getBytes(StandardCharsets.UTF_8)), dtm);
+            new Document(documentReader);
+            fail("Sorry mac");
+        } catch (DeserializationException e) {
+            assertThat(e.getMessage(), e.getMessage().contains("Field 'longfield': Invalid long \"\""), is(true));
+        }
+    }
+
+    @Test
+    public void requireDescriptiveErrorMsgForIntegers() throws Exception {
+        DocumentTypeManager dtm = setupTypes();
+        try {
+            VespaXMLDocumentReader documentReader = new VespaXMLDocumentReader(
+                    new ByteArrayInputStream(("<document id=\"doc:foo:bar\" type=\"doctype\">" +
+                                              "  <intfield></intfield>" +
+                                              "</document>").getBytes(StandardCharsets.UTF_8)), dtm);
+            new Document(documentReader);
+            fail("Sorry mac");
+        } catch (DeserializationException e) {
+            assertThat(e.getMessage(), e.getMessage().contains("Field 'intfield': Invalid integer \"\""), is(true));
+        }
+    }
+
+    @Test
+    public void requireDescriptiveErrorMsgForBytes() throws Exception {
+        DocumentTypeManager dtm = setupTypes();
+        try {
+            VespaXMLDocumentReader documentReader = new VespaXMLDocumentReader(
+                    new ByteArrayInputStream(("<document id=\"doc:foo:bar\" type=\"doctype\">" +
+                                              "  <bytefield></bytefield>" +
+                                              "</document>").getBytes(StandardCharsets.UTF_8)), dtm);
+            new Document(documentReader);
+            fail("Sorry mac");
+        } catch (DeserializationException e) {
+            assertThat(e.getMessage(), e.getMessage().contains("Field 'bytefield': Invalid byte \"\""), is(true));
+        }
+    }
+
+
+
+}
+